StringBuilder split = new StringBuilder();
int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split);
// clear the buffer
split.setLength(0);
if (next >= 0) {
// move over '('
++next;
next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split);
if (next >= 0) {
// move over ')'
++next;
index.set(next);
// found a block
return split.toString();
} else {
throw new ParseException("Unexpected end of block", next);
}
}
// found nothing
return null;
StringBuilder split = new StringBuilder();
int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split);
// clear the buffer
split.setLength(0);
if (next >= 0) {
// move over '('
++next;
next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split);
if (next >= 0) {
// move over ')'
++next;
index.set(next);
// found a block
return split.toString();
} else {
throw new ParseException("Unexpected end of block", next);
}
}
// found nothing
return null;
TODO: sample the generated key/value records, and put the numbers below
TODO: sample the generated key/value records, and put the numbers below
FILE_PATH_CHANGED
setUp()
public void setUp() throws IOException
skip = !(Algorithm.LZO.isSupported());
if (skip) {
System.out.println("Skipped");
}
// TODO: sample the generated key/value records, and put the numbers below
init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo", 2605, 2558);
if (!skip)
super.setUp();
public static String[] split(String str, char escapeChar, char separator)
if (str == null) {
return null;
}
ArrayList strList = new ArrayList();
StringBuilder split = new StringBuilder();
int index = 0;
while ((index = findNext(str, separator, escapeChar, index, split)) >= 0) {
// move over the separator for next search
++index;
strList.add(split.toString());
// reset the buffer
split.setLength(0);
}
strList.add(split.toString());
// remove trailing empty split(s)
// last split
int last = strList.size();
while (--last >= 0 && "".equals(strList.get(last))) {
strList.remove(last);
}
return strList.toArray(new String[strList.size()]);
Some filesystem like HDFS ignore the \"x\" bit if the permission.
Others like localFs does not.
Override the method below if the file system being tested masks our
certain bits for file masks.
Some filesystem like HDFS ignore the \"x\" bit if the permission.
Others like localFs does not.
Override the method below if the file system being tested masks our
certain bits for file masks.
Ignored the mbean itself was not found, which should never happen because we
just accessed it (perhaps something unregistered in-between) but if this
happens just don't output the attribute.
Ignored the mbean itself was not found, which should never happen because we
just accessed it (perhaps something unregistered in-between) but if this
happens just don't output the attribute.
if (!attr.isReadable()) {
return;
}
String attName = attr.getName();
if ("modelerType".equals(attName)) {
return;
}
if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 || attName.indexOf(" ") >= 0) {
return;
}
Object value = null;
try {
value = mBeanServer.getAttribute(oname, attName);
} catch (AttributeNotFoundException e) {
// Ignored the attribute was not found, which should never happen because the bean
// just told us that it has this attribute, but if this happens just don't output
// the attribute.
return;
} catch (MBeanException e) {
// The code inside the attribute getter threw an exception so log it, and
// skip outputting the attribute
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (RuntimeException e) {
// For some reason even with an MBeanException available to them Runtime exceptions
// can still find their way through, so treat them the same as MBeanException
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (ReflectionException e) {
// This happens when the code inside the JMX bean (setter?? from the java docs)
// threw an exception, so log it and skip outputting the attribute
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (InstanceNotFoundException e) {
// Ignored the mbean itself was not found, which should never happen because we
// just accessed it (perhaps something unregistered in-between) but if this
// happens just don't output the attribute.
return;
}
writeAttribute(jg, attName, value);
For some reason even with an MBeanException available to them
Runtime exceptionscan still find their way through, so treat them
the same as MBeanException
For some reason even with an MBeanException available to them
Runtime exceptionscan still find their way through, so treat them
the same as MBeanException
LOG.debug("Listing beans for " + qry);
Set names = null;
names = mBeanServer.queryNames(qry, null);
jg.writeArrayFieldStart("beans");
Iterator it = names.iterator();
while (it.hasNext()) {
ObjectName oname = it.next();
MBeanInfo minfo;
String code = "";
Object attributeinfo = null;
try {
minfo = mBeanServer.getMBeanInfo(oname);
code = minfo.getClassName();
String prs = "";
try {
if ("org.apache.commons.modeler.BaseModelMBean".equals(code)) {
prs = "modelerType";
code = (String) mBeanServer.getAttribute(oname, prs);
}
if (attribute != null) {
prs = attribute;
attributeinfo = mBeanServer.getAttribute(oname, prs);
}
} catch (AttributeNotFoundException e) {
// If the modelerType attribute was not found, the class name is used
// instead.
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
} catch (MBeanException e) {
// The code inside the attribute getter threw an exception so log it,
// and fall back on the class name
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
} catch (RuntimeException e) {
// For some reason even with an MBeanException available to them
// Runtime exceptionscan still find their way through, so treat them
// the same as MBeanException
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
} catch (ReflectionException e) {
// This happens when the code inside the JMX bean (setter?? from the
// java docs) threw an exception, so log it and fall back on the
// class name
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
}
} catch (InstanceNotFoundException e) {
// Ignored for some reason the bean was not found so don't output it
continue;
} catch (IntrospectionException e) {
// This is an internal error, something odd happened with reflection so
// log it and don't output the bean.
LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e);
continue;
} catch (ReflectionException e) {
// This happens when the code inside the JMX bean threw an exception, so
// log it and don't output the bean.
LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e);
continue;
}
jg.writeStartObject();
jg.writeStringField("name", oname.toString());
jg.writeStringField("modelerType", code);
if ((attribute != null) && (attributeinfo == null)) {
jg.writeStringField("result", "ERROR");
jg.writeStringField("message", "No attribute with name " + attribute + " was found.");
jg.writeEndObject();
jg.writeEndArray();
jg.close();
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (attribute != null) {
writeAttribute(jg, attribute, attributeinfo);
} else {
MBeanAttributeInfo[] attrs = minfo.getAttributes();
for (int i = 0; i < attrs.length; i++) {
writeAttribute(jg, oname, attrs[i]);
}
}
jg.writeEndObject();
}
jg.writeEndArray();
TO DO: - more efficient to not split the path, but simply compare
TO DO: - more efficient to not split the path, but simply compare
FILE_PATH_CHANGED
resolve(String, boolean)
// TO DO: - more efficient to not split the path, but simply compare
String[] path = breakIntoPathComponents(p);
if (path.length <= 1) {
// special case for when path is "/"
ResolveResult res = new ResolveResult(ResultKind.isInternalDir, root.InodeDirFs, root.fullPath, SlashPath);
return res;
}
INodeDir curInode = root;
int i;
// ignore first slash
for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) {
INode nextInode = curInode.resolveInternal(path[i]);
if (nextInode == null) {
StringBuilder failedAt = new StringBuilder(path[0]);
for (int j = 1; j <= i; ++j) {
failedAt.append('/').append(path[j]);
}
throw (new FileNotFoundException(failedAt.toString()));
}
if (nextInode instanceof INodeLink) {
final INodeLink link = (INodeLink) nextInode;
final Path remainingPath;
if (i >= path.length - 1) {
remainingPath = SlashPath;
} else {
StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]);
for (int j = i + 2; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult res = new ResolveResult(ResultKind.isExternalDir, link.targetFileSystem, nextInode.fullPath, remainingPath);
return res;
} else if (nextInode instanceof INodeDir) {
curInode = (INodeDir) nextInode;
}
}
// We have resolved to an internal dir in mount table.
Path remainingPath;
if (resolveLastComponent) {
remainingPath = SlashPath;
} else {
// note we have taken care of when path is "/" above
// for internal dirs rem-path does not start with / since the lookup
// that follows will do a children.get(remaningPath) and will have to
// strip-out the initial /
StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
for (int j = i + 1; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult res = new ResolveResult(ResultKind.isInternalDir, curInode.InodeDirFs, curInode.fullPath, remainingPath);
return res;
ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException
NOTE: this is the one place we do NOT want to save the number
of bytes sent (nRemaining here) into lastBytesSent: since we
are resending what we've already sent before, offset is nonzero
in general (only way it could be zero is if it already equals
nRemaining), which would then screw up the offset calculation
_next_ time around. IOW, getRemaining() is in terms of the
original, zero-offset bufferload, so lastBytesSent must be as
well. Cheesy ASCII art:
<------------ m, lastBytesSent ----------->
+===============================================+
buffer: |1111111111|22222222222222222|333333333333| |
+===============================================+
#1: <-- off -->|<-------- nRemaining --------->
#2: <----------- off ----------->|<-- nRem. -->
NOTE: this is the one place we do NOT want to save the number
of bytes sent (nRemaining here) into lastBytesSent: since we
are resending what we've already sent before, offset is nonzero
in general (only way it could be zero is if it already equals
nRemaining), which would then screw up the offset calculation
_next_ time around. IOW, getRemaining() is in terms of the
original, zero-offset bufferload, so lastBytesSent must be as
well. Cheesy ASCII art:
<------------ m, lastBytesSent ----------->
+===============================================+
buffer: |1111111111|22222222222222222|333333333333| |
+===============================================+
#1: <-- off -->|<-------- nRemaining --------->
#2: <----------- off ----------->|<-- nRem. -->
public boolean reportChecksumFailure(Path p, FSDataInputStream in, long inPos, FSDataInputStream sums, long sumsPos)
try {
// canonicalize f
File f = ((RawLocalFileSystem) fs).pathToFile(p).getCanonicalFile();
// find highest writable parent dir of f on the same device
String device = new DF(f, getConf()).getMount();
File parent = f.getParentFile();
File dir = null;
while (parent != null && parent.canWrite() && parent.toString().startsWith(device)) {
dir = parent;
parent = parent.getParentFile();
}
if (dir == null) {
throw new IOException("not able to find the highest writable parent dir");
}
// move the file there
File badDir = new File(dir, "bad_files");
if (!badDir.mkdirs()) {
if (!badDir.isDirectory()) {
throw new IOException("Mkdirs failed to create " + badDir.toString());
}
}
String suffix = "." + rand.nextInt();
File badFile = new File(badDir, f.getName() + suffix);
LOG.warn("Moving bad file " + f + " to " + badFile);
// close it first
in.close();
// rename it
boolean b = f.renameTo(badFile);
if (!b) {
LOG.warn("Ignoring failure of renameTo");
}
// move checksum file too
File checkFile = ((RawLocalFileSystem) fs).pathToFile(getChecksumFile(p));
b = checkFile.renameTo(new File(badDir, checkFile.getName() + suffix));
if (!b) {
LOG.warn("Ignoring failure of renameTo");
}
} catch (IOException e) {
LOG.warn("Error moving bad file " + p + ": " + e);
}
return false;
//
// Done with local copy
//
backupStream.close();
//
// Send it to S3
//
// TODO: Use passed in Progressable to report progress.
nextBlockOutputStream();
store.storeBlock(nextBlock, backupFile);
internalClose();
//
// Delete local backup, start new one
//
boolean b = backupFile.delete();
if (!b) {
LOG.warn("Ignoring failed delete");
}
backupFile = newBackupFile();
backupStream = new FileOutputStream(backupFile);
bytesWrittenToBlock = 0;
Check if zlib consumed all input buffer
set keepUncompressedBuf properly
zlib consumed all input buffer
Check if zlib consumed all input buffer
set keepUncompressedBuf properly
zlib consumed all input buffer
FILE_PATH_CHANGED
compress(byte[], int, int)
public synchronized int compress(byte[] b, int off, int len) throws IOException
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
int n = 0;
// Check if there is compressed data
n = compressedDirectBuf.remaining();
if (n > 0) {
n = Math.min(n, len);
((ByteBuffer) compressedDirectBuf).get(b, off, n);
return n;
}
// Re-initialize the zlib's output direct buffer
compressedDirectBuf.rewind();
compressedDirectBuf.limit(directBufferSize);
// Compress data
n = deflateBytesDirect();
compressedDirectBuf.limit(n);
// Check if zlib consumed all input buffer
// set keepUncompressedBuf properly
if (uncompressedDirectBufLen <= 0) {
// zlib consumed all input buffer
keepUncompressedBuf = false;
uncompressedDirectBuf.clear();
uncompressedDirectBufOff = 0;
uncompressedDirectBufLen = 0;
} else {
// zlib did not consume all input buffer
keepUncompressedBuf = true;
}
// Get atmost 'len' bytes
n = Math.min(n, len);
((ByteBuffer) compressedDirectBuf).get(b, off, n);
return n;
walk through the list, searching. Not the most efficient way, but this
in intended to be used rarely, so we keep it simple.
As an optimization, we can keep a hashmap of record name to its RTI, for later.
walk through the list, searching. Not the most efficient way, but this
in intended to be used rarely, so we keep it simple.
As an optimization, we can keep a hashmap of record name to its RTI, for later.
FILE_PATH_CHANGED
findStruct(String)
// walk through the list, searching. Not the most efficient way, but this
// in intended to be used rarely, so we keep it simple.
// As an optimization, we can keep a hashmap of record name to its RTI, for later.
for (FieldTypeInfo ti : typeInfos) {
if ((0 == ti.getFieldID().compareTo(name)) && (ti.getTypeID().getTypeVal() == RIOType.STRUCT)) {
return (StructTypeID) ti.getTypeID();
}
}
return null;
// Do the authorization
if (HttpServer.hasAdministratorAccess(getServletContext(), request, response)) {
// Authorization is done. Just call super.
super.doGet(request, response);
}
Now another magic 48-bit number, 0x177245385090, to indicate the end
of the last block. (sqrt(pi), if you want to know. I did want to use
e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me
to feel statistically comfortable. Call me paranoid.)
Now another magic 48-bit number, 0x177245385090, to indicate the end
of the last block. (sqrt(pi), if you want to know. I did want to use
e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me
to feel statistically comfortable. Call me paranoid.)
FILE_PATH_CHANGED
hbMakeCodeLengths(char[], int[], int, int)
protected static void hbMakeCodeLengths(char[] len, int[] freq, int alphaSize, int maxLen)
/*
* Nodes and heap entries run from 1. Entry 0 for both the heap and
* nodes is a sentinel.
*/
final int[] heap = new int[MAX_ALPHA_SIZE * 2];
final int[] weight = new int[MAX_ALPHA_SIZE * 2];
final int[] parent = new int[MAX_ALPHA_SIZE * 2];
for (int i = alphaSize; --i >= 0; ) {
weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8;
}
for (boolean tooLong = true; tooLong; ) {
tooLong = false;
int nNodes = alphaSize;
int nHeap = 0;
heap[0] = 0;
weight[0] = 0;
parent[0] = -2;
for (int i = 1; i <= alphaSize; i++) {
parent[i] = -1;
nHeap++;
heap[nHeap] = i;
int zz = nHeap;
int tmp = heap[zz];
while (weight[tmp] < weight[heap[zz >> 1]]) {
heap[zz] = heap[zz >> 1];
zz >>= 1;
}
heap[zz] = tmp;
}
// assert (nHeap < (MAX_ALPHA_SIZE + 2)) : nHeap;
while (nHeap > 1) {
int n1 = heap[1];
heap[1] = heap[nHeap];
nHeap--;
int yy = 0;
int zz = 1;
int tmp = heap[1];
while (true) {
yy = zz << 1;
if (yy > nHeap) {
break;
}
if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) {
yy++;
}
if (weight[tmp] < weight[heap[yy]]) {
break;
}
heap[zz] = heap[yy];
zz = yy;
}
heap[zz] = tmp;
int n2 = heap[1];
heap[1] = heap[nHeap];
nHeap--;
yy = 0;
zz = 1;
tmp = heap[1];
while (true) {
yy = zz << 1;
if (yy > nHeap) {
break;
}
if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) {
yy++;
}
if (weight[tmp] < weight[heap[yy]]) {
break;
}
heap[zz] = heap[yy];
zz = yy;
}
heap[zz] = tmp;
nNodes++;
parent[n1] = parent[n2] = nNodes;
final int weight_n1 = weight[n1];
final int weight_n2 = weight[n2];
weight[nNodes] = (((weight_n1 & 0xffffff00) + (weight_n2 & 0xffffff00)) | (1 + (((weight_n1 & 0x000000ff) > (weight_n2 & 0x000000ff)) ? (weight_n1 & 0x000000ff) : (weight_n2 & 0x000000ff))));
parent[nNodes] = -1;
nHeap++;
heap[nHeap] = nNodes;
tmp = 0;
zz = nHeap;
tmp = heap[zz];
final int weight_tmp = weight[tmp];
while (weight_tmp < weight[heap[zz >> 1]]) {
heap[zz] = heap[zz >> 1];
zz >>= 1;
}
heap[zz] = tmp;
}
// assert (nNodes < (MAX_ALPHA_SIZE * 2)) : nNodes;
for (int i = 1; i <= alphaSize; i++) {
int j = 0;
int k = i;
for (int parent_k; (parent_k = parent[k]) >= 0; ) {
k = parent_k;
j++;
}
len[i - 1] = (char) j;
if (j > maxLen) {
tooLong = true;
}
}
if (tooLong) {
for (int i = 1; i < alphaSize; i++) {
int j = weight[i] >> 8;
j = 1 + (j >> 1);
weight[i] = j << 8;
}
}
}
public static int writeString(DataOutput out, String s) throws IOException
if (s.length() > 0xffff / 3) {
// maybe too long
LOG.warn("truncating long string: " + s.length() + " chars, starting with " + s.substring(0, 20));
s = s.substring(0, 0xffff / 3);
}
int len = utf8Length(s);
if (// double-check length
len > 0xffff)
throw new IOException("string too long!");
out.writeShort(len);
writeChars(out, s, 0, s.length());
return len;
this is very ugly, but needed to avoid breaking hdfs tests...
if a path has no authority, then the FileStatus from globStatus
will add the \"-fs\" authority into the path, so we need to sub
it back out to satisfy the tests
this is very ugly, but needed to avoid breaking hdfs tests...
if a path has no authority, then the FileStatus from globStatus
will add the \"-fs\" authority into the path, so we need to sub
it back out to satisfy the tests
FILE_PATH_CHANGED
expandAsGlob(String, Configuration)
public static PathData[] expandAsGlob(String pattern, Configuration conf) throws IOException
Path globPath = new Path(pattern);
FileSystem fs = globPath.getFileSystem(conf);
FileStatus[] stats = fs.globStatus(globPath);
PathData[] items = null;
if (stats == null) {
// not a glob & file not found, so add the path with a null stat
items = new PathData[] { new PathData(fs, pattern, null) };
} else if (// this is very ugly, but needed to avoid breaking hdfs tests...
// if a path has no authority, then the FileStatus from globStatus
// will add the "-fs" authority into the path, so we need to sub
// it back out to satisfy the tests
stats.length == 1 && stats[0].getPath().equals(fs.makeQualified(globPath))) {
// if the fq path is identical to the pattern passed, use the pattern
// to initialize the string value
items = new PathData[] { new PathData(fs, pattern, stats[0]) };
} else {
// convert stats to PathData
items = new PathData[stats.length];
int i = 0;
for (FileStatus stat : stats) {
items[i++] = new PathData(fs, stat);
}
}
return items;
TODO: DFSAdmin subclasses FsShell so need to protect the command
registration. This class should morph into a base class for
commands, and then this method can be abstract
TODO: DFSAdmin subclasses FsShell so need to protect the command
registration. This class should morph into a base class for
commands, and then this method can be abstract
Ideally we should wait after transferTo returns 0. But because of
a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988),
which throws an exception instead of returning 0, we wait for the
channel to be writable before writing to it. If you ever see
IOException with message \"Resource temporarily unavailable\"
thrown here, please let us know.
Once we move to JAVA SE 7, wait should be moved to correct place.
Ideally we should wait after transferTo returns 0. But because of
a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988),
which throws an exception instead of returning 0, we wait for the
channel to be writable before writing to it. If you ever see
IOException with message \"Resource temporarily unavailable\"
thrown here, please let us know.
Once we move to JAVA SE 7, wait should be moved to correct place.
FILE_PATH_CHANGED
transferToFully(FileChannel, long, int)
public void transferToFully(FileChannel fileCh, long position, int count) throws IOException
while (count > 0) {
/*
* Ideally we should wait after transferTo returns 0. But because of
* a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988),
* which throws an exception instead of returning 0, we wait for the
* channel to be writable before writing to it. If you ever see
* IOException with message "Resource temporarily unavailable"
* thrown here, please let us know.
*
* Once we move to JAVA SE 7, wait should be moved to correct place.
*/
waitForWritable();
int nTransfered = (int) fileCh.transferTo(position, count, getChannel());
if (nTransfered == 0) {
// check if end of file is reached.
if (position >= fileCh.size()) {
throw new EOFException("EOF Reached. file size is " + fileCh.size() + " and " + count + " more bytes left to be " + "transfered.");
}
// otherwise assume the socket is full.
// waitForWritable(); // see comment above.
} else if (nTransfered < 0) {
throw new IOException("Unexpected return of " + nTransfered + " from transferTo()");
} else {
position += nTransfered;
count -= nTransfered;
}
}
public FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum, boolean createParent) throws IOException
checkPath(f);
// Default impl assumes that permissions do not matter
// calling the regular create is good enough.
// FSs that implement permissions should override this.
if (!createParent) {
// parent must exist.
// since this.create makes parent dirs automatically
// we must throw exception if parent does not exist.
final FileStatus stat = getFileStatus(f.getParent());
if (stat == null) {
throw new FileNotFoundException("Missing parent:" + f);
}
if (!stat.isDirectory()) {
throw new ParentNotDirectoryException("parent is not a dir:" + f);
}
// parent does exist - go ahead with create of file.
}
return fsImpl.primitiveCreate(f, absolutePermission, flag, bufferSize, replication, blockSize, progress, bytesPerChecksum);
long length = ftpFile.getSize();
boolean isDir = ftpFile.isDirectory();
int blockReplication = 1;
// Using default block size since there is no way in FTP client to know of
// block sizes on server. The assumption could be less than ideal.
long blockSize = DEFAULT_BLOCK_SIZE;
long modTime = ftpFile.getTimestamp().getTimeInMillis();
long accessTime = 0;
FsPermission permission = getPermissions(ftpFile);
String user = ftpFile.getUser();
String group = ftpFile.getGroup();
Path filePath = new Path(parentPath, ftpFile.getName());
return new FileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, permission, user, group, filePath.makeQualified(this));
public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException
long startTime = System.currentTimeMillis();
IOException ioe;
while (true) {
try {
return getProtocolProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout);
} catch (ConnectException se) {
// namenode has not been started
LOG.info("Server at " + addr + " not available yet, Zzzzz...");
ioe = se;
} catch (SocketTimeoutException te) {
// namenode is busy
LOG.info("Problem connecting to server: " + addr);
ioe = te;
} catch (NoRouteToHostException nrthe) {
// perhaps a VIP is failing over
LOG.info("No route to host for server: " + addr);
ioe = nrthe;
}
// check if timed out
if (System.currentTimeMillis() - timeout >= startTime) {
throw ioe;
}
// wait for retry
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
// IGNORE
}
}
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFsFileStatus that works around.
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFsFileStatus that works around.
FILE_PATH_CHANGED
readOnlyMountTable(String, String)
static AccessControlException readOnlyMountTable(final String operation, final String p)
return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
public void testDeserialization() throws IOException
// Create a test BlockLocation
String[] names = { "one", "two" };
String[] hosts = { "three", "four" };
String[] topologyPaths = { "five", "six" };
long offset = 25l;
long length = 55l;
BlockLocation bl = new BlockLocation(names, hosts, topologyPaths, offset, length);
DataOutputBuffer dob = new DataOutputBuffer();
// Serialize it
try {
bl.write(dob);
} catch (IOException e) {
fail("Unable to serialize data: " + e.getMessage());
}
byte[] bytes = dob.getData();
DataInput da = new DataInputStream(new ByteArrayInputStream(bytes));
// Try to re-create the BlockLocation the same way as is done during
// deserialization
BlockLocation bl2 = new BlockLocation();
try {
bl2.readFields(da);
} catch (IOException e) {
fail("Unable to deserialize BlockLocation: " + e.getMessage());
}
// Check that we got back what we started with
verifyDeserialization(bl2.getHosts(), hosts);
verifyDeserialization(bl2.getNames(), names);
verifyDeserialization(bl2.getTopologyPaths(), topologyPaths);
assertEquals(bl2.getOffset(), offset);
assertEquals(bl2.getLength(), length);
Check that every permission has its sticky bit represented correctly
Check that every permission has its sticky bit represented correctly
FILE_PATH_CHANGED
testStickyBitToString()
public void testStickyBitToString()
// Check that every permission has its sticky bit represented correctly
for (boolean sb : new boolean[] { false, true }) {
for (FsAction u : FsAction.values()) {
for (FsAction g : FsAction.values()) {
for (FsAction o : FsAction.values()) {
FsPermission f = new FsPermission(u, g, o, sb);
if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('t', f.toString().charAt(8));
else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE))
assertEquals('T', f.toString().charAt(8));
else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('x', f.toString().charAt(8));
else
assertEquals('-', f.toString().charAt(8));
}
}
}
}
if (item.stat.isDirectory()) {
// TODO: handle this
throw new PathIsDirectoryException(item.toString());
}
if (item.stat.getLen() != 0) {
throw new PathIOException(item.toString(), "Not a zero-length file");
}
touchz(item);
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
FILE_PATH_CHANGED
testOldFormat()
public void testOldFormat() throws IOException
// Make sure we still correctly write the old format if desired.
// Write the data array with old ObjectWritable API
// which will set allowCompactArrays false.
ObjectWritable.writeObject(out, i, i.getClass(), null);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
@SuppressWarnings("deprecation")
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not labelled as an array of int", i.getClass().getName(), className);
int length = in.readInt();
assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not expected length", i.length, length);
int[] readValue = new int[length];
try {
for (int i = 0; i < length; i++) {
readValue[i] = (int) ((Integer) ObjectWritable.readObject(in, null));
}
} catch (Exception e) {
fail("The int[] written by ObjectWritable as a non-compact array " + "was corrupted. Failed to correctly read int[] of length " + length + ". Got exception:\n" + StringUtils.stringifyException(e));
}
assertTrue("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.", Arrays.equals(i, readValue));
Read the APW object as written by ObjectWritable, but
\"going around\" ObjectWritable
Read the APW object as written by ObjectWritable, but
\"going around\" ObjectWritable
FILE_PATH_CHANGED
testObjectLabeling()
public void testObjectLabeling() throws IOException
// Do a few tricky experiments to make sure things are being written
// the way we expect
// Write the data array with ObjectWritable
// which will indirectly write it using APW.Internal
ObjectWritable.writeObject(out, i, i.getClass(), null, true);
// Write the corresponding APW directly with ObjectWritable
ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i);
ObjectWritable.writeObject(out, apw, apw.getClass(), null, true);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className);
ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal();
apwi.readFields(in);
assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get())));
// Read the APW object as written by ObjectWritable, but
// "going around" ObjectWritable
String declaredClassName = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName);
className = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className);
ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable();
apw2.readFields(in);
assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
FILE_PATH_CHANGED
testObjectLabeling()
public void testObjectLabeling() throws IOException
// Do a few tricky experiments to make sure things are being written
// the way we expect
// Write the data array with ObjectWritable
// which will indirectly write it using APW.Internal
ObjectWritable.writeObject(out, i, i.getClass(), null, true);
// Write the corresponding APW directly with ObjectWritable
ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i);
ObjectWritable.writeObject(out, apw, apw.getClass(), null, true);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className);
ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal();
apwi.readFields(in);
assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get())));
// Read the APW object as written by ObjectWritable, but
// "going around" ObjectWritable
String declaredClassName = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName);
className = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className);
ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable();
apw2.readFields(in);
assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
public FileMetadata retrieveMetadata(String key) throws IOException
try {
S3Object object = s3Service.getObjectDetails(bucket, key);
return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime());
} catch (S3ServiceException e) {
// Following is brittle. Is there a better way?
if (e.getMessage().contains("ResponseCode=404")) {
return null;
}
handleServiceException(e);
// never returned - keep compiler happy
return null;
}
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
if (name == null) {
LOG.warn("Metric was emitted with no name.");
return;
} else if (value == null) {
LOG.warn("Metric name " + name + " was emitted with a null value.");
return;
} else if (type == null) {
LOG.warn("Metric name " + name + ", value " + value + " has no type.");
return;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + ", slope " + gSlope.name() + " from hostname " + getHostName());
}
// The following XDR recipe was done through a careful reading of
// gm_protocol.x in Ganglia 3.1 and carefully examining the output of
// the gmetric utility with strace.
// First we send out a metadata message
// metric_id = metadata_msg
xdr_int(128);
// hostname
xdr_string(getHostName());
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// metric type
xdr_string(type);
// metric name
xdr_string(name);
// units
xdr_string(gConf.getUnits());
// slope
xdr_int(gSlope.ordinal());
// tmax, the maximum time between metrics
xdr_int(gConf.getTmax());
// dmax, the maximum data value
xdr_int(gConf.getDmax());
xdr_int(1);
/*Num of the entries in extra_value field for
Ganglia 3.1.x*/
xdr_string("GROUP");
/*Group attribute*/
xdr_string(groupName);
/*Group value*/
// send the metric to Ganglia hosts
emitToGangliaHosts();
// Now we send out a message with the actual value.
// Technically, we only need to send out the metadata message once for
// each metric, but I don't want to have to record which metrics we did and
// did not send.
// we are sending a string value
xdr_int(133);
// hostName
xdr_string(getHostName());
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// format field
xdr_string("%s");
// metric value
xdr_string(value);
// send the metric to Ganglia hosts
emitToGangliaHosts();
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
if (name == null) {
LOG.warn("Metric was emitted with no name.");
return;
} else if (value == null) {
LOG.warn("Metric name " + name + " was emitted with a null value.");
return;
} else if (type == null) {
LOG.warn("Metric name " + name + ", value " + value + " has no type.");
return;
}
LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName);
String units = getUnits(name);
if (units == null) {
LOG.warn("Metric name " + name + ", value " + value + " had 'null' units");
units = "";
}
int slope = getSlope(name);
int tmax = getTmax(name);
int dmax = getDmax(name);
offset = 0;
String groupName = name.substring(0, name.lastIndexOf("."));
// The following XDR recipe was done through a careful reading of
// gm_protocol.x in Ganglia 3.1 and carefully examining the output of
// the gmetric utility with strace.
// First we send out a metadata message
// metric_id = metadata_msg
xdr_int(128);
// hostname
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// metric type
xdr_string(type);
// metric name
xdr_string(name);
// units
xdr_string(units);
// slope
xdr_int(slope);
// tmax, the maximum time between metrics
xdr_int(tmax);
// dmax, the maximum data value
xdr_int(dmax);
xdr_int(1);
/*Num of the entries in extra_value field for
Ganglia 3.1.x*/
xdr_string("GROUP");
/*Group attribute*/
xdr_string(groupName);
/*Group value*/
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
// Now we send out a message with the actual value.
// Technically, we only need to send out the metadata message once for
// each metric, but I don't want to have to record which metrics we did and
// did not send.
offset = 0;
// we are sending a string value
xdr_int(133);
// hostName
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// format field
xdr_string("%s");
// metric value
xdr_string(value);
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
Default impl is to assume that permissions do not matter and hence
calling the regular mkdirs is good enough.
FSs that implement permissions should override this.
Default impl is to assume that permissions do not matter and hence
calling the regular mkdirs is good enough.
FSs that implement permissions should override this.
Default impl assumes that permissions do not matter and
nor does the bytesPerChecksum hence
calling the regular create is good enough.
FSs that implement permissions should override this.
Default impl assumes that permissions do not matter and
nor does the bytesPerChecksum hence
calling the regular create is good enough.
FSs that implement permissions should override this.
The following if clause handles the following case:
Assume the following scenario in BZip2 compressed stream where
. represent compressed data.
.....[48 bit Block].....[48 bit Block].....[48 bit Block]...
........................[47 bits][1 bit].....[48 bit Block]...
................................^[Assume a Byte alignment here]
........................................^^[current position of stream]
.....................^^[We go back 10 Bytes in stream and find a Block marker]
........................................^^[We align at wrong position!]
...........................................................^^[While this pos is correct]
The following if clause handles the following case:
Assume the following scenario in BZip2 compressed stream where
. represent compressed data.
.....[48 bit Block].....[48 bit Block].....[48 bit Block]...
........................[47 bits][1 bit].....[48 bit Block]...
................................^[Assume a Byte alignment here]
........................................^^[current position of stream]
.....................^^[We go back 10 Bytes in stream and find a Block marker]
........................................^^[We align at wrong position!]
...........................................................^^[While this pos is correct]
FILE_PATH_CHANGED
createOutputStream(OutputStream)
public CompressionOutputStream createOutputStream(OutputStream out) throws IOException
public boolean mkdirs(Path dir, FsPermission permission) throws AccessControlException, FileAlreadyExistsException
if (theInternalDir.isRoot & dir == null) {
throw new FileAlreadyExistsException("/ already exits");
}
// Note dir starts with /
if (theInternalDir.children.containsKey(dir.toString().substring(1))) {
// this is the stupid semantics of FileSystem
return true;
}
throw readOnlyMountTable("mkdirs", dir);
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFileSystemFileStatus that
works around.
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFileSystemFileStatus that
works around.
FILE_PATH_CHANGED
readOnlyMountTable(String, String)
static AccessControlException readOnlyMountTable(final String operation, final String p)
return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
TODO: if the user wants the trash to be used but there is any
problem (ie. creating the trash dir, moving the item to be deleted,
etc), then the path will just be deleted because moveToTrash returns
false and it falls thru to fs.delete. this doesn't seem right
TODO: if the user wants the trash to be used but there is any
problem (ie. creating the trash dir, moving the item to be deleted,
etc), then the path will just be deleted because moveToTrash returns
false and it falls thru to fs.delete. this doesn't seem right
FILE_PATH_CHANGED
registerCommands(CommandFactory)
public static void registerCommands(CommandFactory factory)
public synchronized void refresh(Configuration conf, PolicyProvider provider)
// Get the system property 'hadoop.policy.file'
String policyFile = System.getProperty("hadoop.policy.file", HADOOP_POLICY_FILE);
// Make a copy of the original config, and load the policy file
Configuration policyConf = new Configuration(conf);
policyConf.addResource(policyFile);
final Map, AccessControlList> newAcls = new IdentityHashMap, AccessControlList>();
// Parse the config file
Service[] services = provider.getServices();
if (services != null) {
for (Service service : services) {
AccessControlList acl = new AccessControlList(policyConf.get(service.getServiceKey(), AccessControlList.WILDCARD_ACL_VALUE));
newAcls.put(service.getProtocol(), acl);
}
}
// Flip to the newly parsed permissions
protocolToAcl = newAcls;
Read characters into a portion of an array, reading from the underlying
stream at most once if necessary.
Read characters into a portion of an array, reading from the underlying
stream at most once if necessary.
FILE_PATH_CHANGED
read1(byte[], int, int)
private int read1(byte[] b, int off, int len) throws IOException
int avail = count - pos;
if (avail <= 0) {
if (len >= maxChunkSize) {
// read a chunk to user buffer directly; avoid one copy
int nread = readChecksumChunk(b, off, len);
return nread;
} else {
// read a chunk into the local buffer
fill();
if (count <= 0) {
return -1;
} else {
avail = count;
}
}
}
// copy content of the local buffer to the user buffer
int cnt = (avail < len) ? avail : len;
System.arraycopy(buf, pos, b, off, cnt);
pos += cnt;
return cnt;
FIXME? Inflater docs say: 'it is also necessary to provide an extra
\"dummy\" byte as input. This is required by the ZLIB native
library in order to support certain optimizations.' However,
this does not appear to be true, and in any case, it's not
entirely clear where the byte should go or what its value
should be. Perhaps it suffices to have some deflated bytes
in the first buffer load? (But how else would one do it?)
FIXME? Inflater docs say: 'it is also necessary to provide an extra
\"dummy\" byte as input. This is required by the ZLIB native
library in order to support certain optimizations.' However,
this does not appear to be true, and in any case, it's not
entirely clear where the byte should go or what its value
should be. Perhaps it suffices to have some deflated bytes
in the first buffer load? (But how else would one do it?)
FILE_PATH_CHANGED
needsInput()
public synchronized boolean needsInput()
if (state == GzipStateLabel.DEFLATE_STREAM) {
// most common case
return inflater.needsInput();
}
// see userBufLen comment at top of decompress(); currently no need to
// verify userBufLen <= 0
return (state != GzipStateLabel.FINISHED);
this is an unexpected condition, so dump the whole exception since
it's probably a nasty internal error where the backtrace would be
useful
this is an unexpected condition, so dump the whole exception since
it's probably a nasty internal error where the backtrace would be
useful
FILE_PATH_CHANGED
displayError(Exception)
public void displayError(Exception e)
// build up a list of exceptions that occurred
exceptions.add(e);
String errorMessage = e.getLocalizedMessage();
if (errorMessage == null) {
// this is an unexpected condition, so dump the whole exception since
// it's probably a nasty internal error where the backtrace would be
// useful
errorMessage = StringUtils.stringifyException(e);
LOG.debug(errorMessage);
} else {
errorMessage = errorMessage.split("\n", 2)[0];
}
displayError(errorMessage);
public void testLogGroupRecoveryInProgress() throws IOException
String[] paths = new String[] { "/foo1/current/" + getInProgressEditsFileName(123), "/foo2/current/" + getInProgressEditsFileName(123), "/foo3/current/" + getInProgressEditsFileName(123) };
FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector();
inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[0]));
inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[1]));
inspector.inspectDirectory(mockDirectoryWithEditLogs(paths[2]));
// Inject spies to return the valid counts we would like to see
mockLogValidation(inspector, paths[0], 2000);
mockLogValidation(inspector, paths[1], 2000);
mockLogValidation(inspector, paths[2], 1000);
LogGroup lg = inspector.logGroups.get(123L);
assertEquals(3, lg.logs.size());
lg.planRecovery();
// Check that the short one was marked corrupt
assertFalse(lg.logs.get(0).isCorrupt());
assertFalse(lg.logs.get(1).isCorrupt());
assertTrue(lg.logs.get(2).isCorrupt());
// Calling recover should move it aside
FoundEditLog badLog = lg.logs.get(2);
Mockito.doNothing().when(badLog).moveAsideCorruptFile();
Mockito.doNothing().when(lg.logs.get(0)).finalizeLog();
Mockito.doNothing().when(lg.logs.get(1)).finalizeLog();
lg.recover();
Mockito.verify(badLog).moveAsideCorruptFile();
Mockito.verify(lg.logs.get(0)).finalizeLog();
Mockito.verify(lg.logs.get(1)).finalizeLog();
public void testCountValidTransactions() throws IOException
File testDir = new File(TEST_DIR, "testCountValidTransactions");
File logFile = new File(testDir, NNStorage.getInProgressEditsFileName(1));
// Create a log file, and return the offsets at which each
// transaction starts.
FSEditLog fsel = null;
final int NUM_TXNS = 30;
SortedMap offsetToTxId = Maps.newTreeMap();
try {
fsel = FSImageTestUtil.createStandaloneEditLog(testDir);
fsel.open();
assertTrue("should exist: " + logFile, logFile.exists());
for (int i = 0; i < NUM_TXNS; i++) {
long trueOffset = getNonTrailerLength(logFile);
long thisTxId = fsel.getLastWrittenTxId() + 1;
offsetToTxId.put(trueOffset, thisTxId);
System.err.println("txid " + thisTxId + " at offset " + trueOffset);
fsel.logDelete("path" + i, i);
fsel.logSync();
}
} finally {
if (fsel != null) {
fsel.close();
}
}
// The file got renamed when the log was closed.
logFile = testDir.listFiles()[0];
long validLength = getNonTrailerLength(logFile);
// Make sure that uncorrupted log has the expected length and number
// of transactions.
EditLogValidation validation = FSEditLogLoader.validateEditLog(logFile);
assertEquals(NUM_TXNS + 2, validation.numTransactions);
assertEquals(validLength, validation.validLength);
// Back up the uncorrupted log
File logFileBak = new File(testDir, logFile.getName() + ".bak");
Files.copy(logFile, logFileBak);
// Corrupt the log file in various ways for each txn
for (Map.Entry entry : offsetToTxId.entrySet()) {
long txOffset = entry.getKey();
long txid = entry.getValue();
// Restore backup, truncate the file exactly before the txn
Files.copy(logFileBak, logFile);
truncateFile(logFile, txOffset);
validation = FSEditLogLoader.validateEditLog(logFile);
assertEquals("Failed when truncating to length " + txOffset, txid - 1, validation.numTransactions);
assertEquals(txOffset, validation.validLength);
// Restore backup, truncate the file with one byte in the txn,
// also isn't valid
Files.copy(logFileBak, logFile);
truncateFile(logFile, txOffset + 1);
validation = FSEditLogLoader.validateEditLog(logFile);
assertEquals("Failed when truncating to length " + (txOffset + 1), txid - 1, validation.numTransactions);
assertEquals(txOffset, validation.validLength);
// Restore backup, corrupt the txn opcode
Files.copy(logFileBak, logFile);
corruptByteInFile(logFile, txOffset);
validation = FSEditLogLoader.validateEditLog(logFile);
assertEquals("Failed when corrupting txn opcode at " + txOffset, txid - 1, validation.numTransactions);
assertEquals(txOffset, validation.validLength);
// Restore backup, corrupt a byte a few bytes into the txn
Files.copy(logFileBak, logFile);
corruptByteInFile(logFile, txOffset + 5);
validation = FSEditLogLoader.validateEditLog(logFile);
assertEquals("Failed when corrupting txn data at " + (txOffset + 5), txid - 1, validation.numTransactions);
assertEquals(txOffset, validation.validLength);
}
// Corrupt the log at every offset to make sure that validation itself
// never throws an exception, and that the calculated lengths are monotonically
// increasing
long prevNumValid = 0;
for (long offset = 0; offset < validLength; offset++) {
Files.copy(logFileBak, logFile);
corruptByteInFile(logFile, offset);
EditLogValidation val = FSEditLogLoader.validateEditLog(logFile);
assertTrue(val.numTransactions >= prevNumValid);
prevNumValid = val.numTransactions;
}
Configuration conf = getConf();
NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
DFSTestUtil.formatNameNode(conf);
FSNamesystem fsn = new FSNamesystem(conf);
// Replace the FSImage with a spy
FSImage originalImage = fsn.dir.fsImage;
NNStorage storage = originalImage.getStorage();
NNStorage spyStorage = spy(storage);
originalImage.storage = spyStorage;
FSImage spyImage = spy(originalImage);
fsn.dir.fsImage = spyImage;
// should we expect the save operation to fail
boolean shouldFail = false;
// inject fault
switch(fault) {
case SAVE_SECOND_FSIMAGE_RTE:
// The spy throws a RuntimeException when writing to the second directory
doAnswer(new FaultySaveImage(true)).when(spyImage).saveFSImage((StorageDirectory) anyObject(), anyLong());
shouldFail = false;
break;
case SAVE_SECOND_FSIMAGE_IOE:
// The spy throws an IOException when writing to the second directory
doAnswer(new FaultySaveImage(false)).when(spyImage).saveFSImage((StorageDirectory) anyObject(), anyLong());
shouldFail = false;
break;
case SAVE_ALL_FSIMAGES:
// The spy throws IOException in all directories
doThrow(new RuntimeException("Injected")).when(spyImage).saveFSImage((StorageDirectory) anyObject(), anyLong());
shouldFail = true;
break;
case WRITE_STORAGE_ALL:
// The spy throws an exception before writing any VERSION files
doThrow(new RuntimeException("Injected")).when(spyStorage).writeAll();
shouldFail = true;
break;
case WRITE_STORAGE_ONE:
// The spy throws on exception on one particular storage directory
doAnswer(new FaultySaveImage(true)).when(spyStorage).writeProperties((StorageDirectory) anyObject());
// TODO: unfortunately this fails -- should be improved.
// See HDFS-2173.
shouldFail = true;
break;
}
try {
doAnEdit(fsn, 1);
// Save namespace - this may fail, depending on fault injected
fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
try {
fsn.saveNamespace();
if (shouldFail) {
fail("Did not fail!");
}
} catch (Exception e) {
if (!shouldFail) {
throw e;
} else {
LOG.info("Test caught expected exception", e);
}
}
fsn.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
// Should still be able to perform edits
doAnEdit(fsn, 2);
// Now shut down and restart the namesystem
originalImage.close();
fsn.close();
fsn = null;
// Start a new namesystem, which should be able to recover
// the namespace from the previous incarnation.
fsn = new FSNamesystem(conf);
// Make sure the image loaded including our edits.
checkEditExists(fsn, 1);
checkEditExists(fsn, 2);
} finally {
if (fsn != null) {
fsn.close();
}
}
We appear to have missed some transactions -- the NN probably
lost contact with us temporarily. So, mark the current segment
as aborted.
SATD_ADDED
namenodeStartedLogSegment(long)
LOG.info("NameNode started a new log segment at txid " + txid);
if (editLog.isOpen()) {
if (editLog.getLastWrittenTxId() == txid - 1) {
// We are in sync with the NN, so end and finalize the current segment
editLog.endCurrentLogSegment(false);
} else {
// We appear to have missed some transactions -- the NN probably
// lost contact with us temporarily. So, mark the current segment
// as aborted.
LOG.warn("NN started new log segment at txid " + txid + ", but BN had only written up to txid " + editLog.getLastWrittenTxId() + "in the log segment starting at " + editLog.getCurSegmentTxId() + ". Aborting this " + "log segment.");
editLog.abortCurrentLogSegment();
}
}
editLog.setNextTxId(txid);
editLog.startLogSegment(txid, false);
if (bnState == BNState.DROP_UNTIL_NEXT_ROLL) {
setState(BNState.JOURNAL_ONLY);
}
if (stopApplyingEditsOnNextRoll) {
if (bnState == BNState.IN_SYNC) {
LOG.info("Stopped applying edits to prepare for checkpoint.");
setState(BNState.JOURNAL_ONLY);
}
stopApplyingEditsOnNextRoll = false;
notifyAll();
}
String fileid = GetImageServlet.getParamStringToPutImage(txid, imageListenAddress, storage);
// this doesn't directly upload an image, but rather asks the NN
// to connect back to the 2NN to download the specified image.
TransferFsImage.getFileClient(fsName, fileid, null, null, false);
LOG.info("Uploaded image with txid " + txid + " to namenode at " + fsName);
public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException
try {
ServletContext context = getServletContext();
final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context);
final GetImageParams parsedParams = new GetImageParams(request, response);
final Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
if (UserGroupInformation.isSecurityEnabled() && !isValidRequestor(request.getRemoteUser(), conf)) {
response.sendError(HttpServletResponse.SC_FORBIDDEN, "Only Namenode and Secondary Namenode may access this servlet");
LOG.warn("Received non-NN/SNN request for image or edits from " + request.getRemoteHost());
return;
}
String myStorageInfoString = nnImage.getStorage().toColonSeparatedString();
String theirStorageInfoString = parsedParams.getStorageInfoString();
if (theirStorageInfoString != null && !myStorageInfoString.equals(theirStorageInfoString)) {
response.sendError(HttpServletResponse.SC_FORBIDDEN, "This namenode has storage info " + myStorageInfoString + " but the secondary expected " + theirStorageInfoString);
LOG.warn("Received an invalid request file transfer request " + "from a secondary with storage info " + theirStorageInfoString);
return;
}
UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction() {
@Override
public Void run() throws Exception {
if (parsedParams.isGetImage()) {
long txid = parsedParams.getTxId();
File imageFile = nnImage.getStorage().getFsImageName(txid);
if (imageFile == null) {
throw new IOException("Could not find image with txid " + txid);
}
setVerificationHeaders(response, imageFile);
// send fsImage
TransferFsImage.getFileServer(response.getOutputStream(), imageFile, getThrottler(conf));
} else if (parsedParams.isGetEdit()) {
long startTxId = parsedParams.getStartTxId();
long endTxId = parsedParams.getEndTxId();
File editFile = nnImage.getStorage().findFinalizedEditsFile(startTxId, endTxId);
setVerificationHeaders(response, editFile);
// send edits
TransferFsImage.getFileServer(response.getOutputStream(), editFile, getThrottler(conf));
} else if (parsedParams.isPutImage()) {
final long txid = parsedParams.getTxId();
if (!currentlyDownloadingCheckpoints.add(txid)) {
throw new IOException("Another checkpointer is already in the process of uploading a" + " checkpoint made at transaction ID " + txid);
}
try {
if (nnImage.getStorage().findImageFile(txid) != null) {
throw new IOException("Another checkpointer already uploaded an checkpoint " + "for txid " + txid);
}
// issue a HTTP get request to download the new fsimage
MD5Hash downloadImageDigest = reloginIfNecessary().doAs(new PrivilegedExceptionAction() {
@Override
public MD5Hash run() throws Exception {
return TransferFsImage.downloadImageToStorage(parsedParams.getInfoServer(), txid, nnImage.getStorage(), true);
}
});
nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest);
// Now that we have a new checkpoint, we might be able to
// remove some old ones.
nnImage.purgeOldStorage();
} finally {
currentlyDownloadingCheckpoints.remove(txid);
}
}
return null;
}
// We may have lost our ticket since the last time we tried to open
// an http connection, so log in just in case.
private UserGroupInformation reloginIfNecessary() throws IOException {
// This method is only called on the NN, therefore it is safe to
// use these key values.
return UserGroupInformation.loginUserFromKeytabAndReturnUGI(SecurityUtil.getServerPrincipal(conf.get(DFSConfigKeys.DFS_NAMENODE_KRB_HTTPS_USER_NAME_KEY), NameNode.getAddress(conf).getHostName()), conf.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
}
});
} catch (Exception ie) {
String errMsg = "GetImage failed. " + StringUtils.stringifyException(ie);
response.sendError(HttpServletResponse.SC_GONE, errMsg);
throw new IOException(errMsg);
} finally {
response.getOutputStream().close();
}
checkNotSaved();
FSDirectory fsDir = sourceNamesystem.dir;
long startTime = now();
//
// Write out data
//
MessageDigest digester = MD5Hash.getDigester();
FileOutputStream fout = new FileOutputStream(newFile);
DigestOutputStream fos = new DigestOutputStream(fout, digester);
DataOutputStream out = new DataOutputStream(fos);
try {
out.writeInt(FSConstants.LAYOUT_VERSION);
out.writeInt(sourceNamesystem.getFSImage().getStorage().getNamespaceID());
out.writeLong(fsDir.rootDir.numItemsInTree());
out.writeLong(sourceNamesystem.getGenerationStamp());
out.writeLong(txid);
// write compression info and set up compressed stream
out = compression.writeHeaderAndWrapStream(fos);
LOG.info("Saving image file " + newFile + " using " + compression);
byte[] byteStore = new byte[4 * FSConstants.MAX_PATH_LENGTH];
ByteBuffer strbuf = ByteBuffer.wrap(byteStore);
// save the root
FSImageSerialization.saveINode2Image(fsDir.rootDir, out);
// save the rest of the nodes
saveImage(strbuf, fsDir.rootDir, out);
// save files under construction
sourceNamesystem.saveFilesUnderConstruction(out);
sourceNamesystem.saveSecretManagerState(out);
strbuf = null;
out.flush();
fout.getChannel().force(true);
} finally {
out.close();
}
saved = true;
// set md5 of the saved image
savedDigest = new MD5Hash(digester.digest());
LOG.info("Image file of size " + newFile.length() + " saved in " + (now() - startTime) / 1000 + " seconds.");
Since we're creating a new UserGroupInformation here, we know that no
future RPC proxies will be able to re-use the same connection. And
usages of this proxy tend to be one-off calls.
This is a temporary fix: callers should really achieve this by using
RPC.stopProxy() on the resulting object, but this is currently not
working in trunk. See the discussion on HDFS-1965.
Since we're creating a new UserGroupInformation here, we know that no
future RPC proxies will be able to re-use the same connection. And
usages of this proxy tend to be one-off calls.
This is a temporary fix: callers should really achieve this by using
RPC.stopProxy() on the resulting object, but this is currently not
working in trunk. See the discussion on HDFS-1965.
public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(DatanodeID datanodeid, Configuration conf, int socketTimeout, LocatedBlock locatedBlock) throws IOException
InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getHost() + ":" + datanodeid.getIpcPort());
if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
ClientDatanodeProtocol.LOG.debug("ClientDatanodeProtocol addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
UserGroupInformation ticket = UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
ticket.addToken(locatedBlock.getBlockToken());
return (ClientDatanodeProtocol) RPC.getProxy(ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
public void sortLocatedBlocks(final String targethost, final List locatedblocks)
// sort the blocks
final DatanodeDescriptor client = getDatanodeByHost(targethost);
for (LocatedBlock b : locatedblocks) {
networktopology.pseudoSortByDistance(client, b.getLocations());
// Move decommissioned datanodes to the bottom
Arrays.sort(b.getLocations(), DFSUtil.DECOM_COMPARATOR);
}
Exception while writing to the client. Connection closure from
the other end is mostly the case and we do not care much about
it. But other things can go wrong, especially in transferTo(),
which we do not want to ignore.
The message parsing below should not be considered as a good
coding example. NEVER do it to drive a program logic. NEVER.
It was done here because the NIO throws an IOException for EPIPE.
SATD_ADDED
sendChunks(ByteBuffer, int, OutputStream)
private int sendChunks(ByteBuffer pkt, int maxChunks, OutputStream out) throws IOException
// Sends multiple chunks in one packet with a single write().
int len = (int) Math.min(endOffset - offset, (((long) bytesPerChecksum) * ((long) maxChunks)));
int numChunks = (len + bytesPerChecksum - 1) / bytesPerChecksum;
int packetLen = len + numChunks * checksumSize + 4;
boolean lastDataPacket = offset + len == endOffset && len > 0;
pkt.clear();
PacketHeader header = new PacketHeader(packetLen, offset, seqno, (len == 0), len);
header.putInBuffer(pkt);
int checksumOff = pkt.position();
int checksumLen = numChunks * checksumSize;
byte[] buf = pkt.array();
if (checksumSize > 0 && checksumIn != null) {
try {
checksumIn.readFully(buf, checksumOff, checksumLen);
} catch (IOException e) {
LOG.warn(" Could not read or failed to veirfy checksum for data" + " at offset " + offset + " for block " + block + " got : " + StringUtils.stringifyException(e));
IOUtils.closeStream(checksumIn);
checksumIn = null;
if (corruptChecksumOk) {
if (checksumOff < checksumLen) {
// Just fill the array with zeros.
Arrays.fill(buf, checksumOff, checksumLen, (byte) 0);
}
} else {
throw e;
}
}
// write in progress that we need to use to get last checksum
if (lastDataPacket && lastChunkChecksum != null) {
int start = checksumOff + checksumLen - checksumSize;
byte[] updatedChecksum = lastChunkChecksum.getChecksum();
if (updatedChecksum != null) {
System.arraycopy(updatedChecksum, 0, buf, start, checksumSize);
}
}
}
int dataOff = checksumOff + checksumLen;
if (blockInPosition < 0) {
// normal transfer
IOUtils.readFully(blockIn, buf, dataOff, len);
if (verifyChecksum) {
int dOff = dataOff;
int cOff = checksumOff;
int dLeft = len;
for (int i = 0; i < numChunks; i++) {
checksum.reset();
int dLen = Math.min(dLeft, bytesPerChecksum);
checksum.update(buf, dOff, dLen);
if (!checksum.compare(buf, cOff)) {
long failedPos = offset + len - dLeft;
throw new ChecksumException("Checksum failed at " + failedPos, failedPos);
}
dLeft -= dLen;
dOff += dLen;
cOff += checksumSize;
}
}
// writing is done below (mainly to handle IOException)
}
try {
if (blockInPosition >= 0) {
// use transferTo(). Checks on out and blockIn are already done.
SocketOutputStream sockOut = (SocketOutputStream) out;
// first write the packet
sockOut.write(buf, 0, dataOff);
// no need to flush. since we know out is not a buffered stream.
sockOut.transferToFully(((FileInputStream) blockIn).getChannel(), blockInPosition, len);
blockInPosition += len;
} else {
// normal transfer
out.write(buf, 0, dataOff + len);
}
} catch (IOException e) {
/* Exception while writing to the client. Connection closure from
* the other end is mostly the case and we do not care much about
* it. But other things can go wrong, especially in transferTo(),
* which we do not want to ignore.
*
* The message parsing below should not be considered as a good
* coding example. NEVER do it to drive a program logic. NEVER.
* It was done here because the NIO throws an IOException for EPIPE.
*/
String ioem = e.getMessage();
if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) {
LOG.error("BlockSender.sendChunks() exception: ", e);
}
throw ioeToSocketException(e);
}
if (throttler != null) {
// rebalancing so throttle
throttler.throttle(packetLen);
}
return len;
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
if (name == null) {
LOG.warn("Metric was emitted with no name.");
return;
} else if (value == null) {
LOG.warn("Metric name " + name + " was emitted with a null value.");
return;
} else if (type == null) {
LOG.warn("Metric name " + name + ", value " + value + " has no type.");
return;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + ", slope " + gSlope.name() + " from hostname " + getHostName());
}
// The following XDR recipe was done through a careful reading of
// gm_protocol.x in Ganglia 3.1 and carefully examining the output of
// the gmetric utility with strace.
// First we send out a metadata message
// metric_id = metadata_msg
xdr_int(128);
// hostname
xdr_string(getHostName());
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// metric type
xdr_string(type);
// metric name
xdr_string(name);
// units
xdr_string(gConf.getUnits());
// slope
xdr_int(gSlope.ordinal());
// tmax, the maximum time between metrics
xdr_int(gConf.getTmax());
// dmax, the maximum data value
xdr_int(gConf.getDmax());
xdr_int(1);
/*Num of the entries in extra_value field for
Ganglia 3.1.x*/
xdr_string("GROUP");
/*Group attribute*/
xdr_string(groupName);
/*Group value*/
// send the metric to Ganglia hosts
emitToGangliaHosts();
// Now we send out a message with the actual value.
// Technically, we only need to send out the metadata message once for
// each metric, but I don't want to have to record which metrics we did and
// did not send.
// we are sending a string value
xdr_int(133);
// hostName
xdr_string(getHostName());
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// format field
xdr_string("%s");
// metric value
xdr_string(value);
// send the metric to Ganglia hosts
emitToGangliaHosts();
Keeps a TreeSet for every named node. Each treeset contains
a list of the blocks that are \"extra\" at that location. We'll
eventually remove these extras.
Mapping: StorageID -> TreeSet
add a block to a under replication queue according to its priority
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
add a block to a under replication queue according to its priority
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
FILE_PATH_CHANGED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
Return the priority of a block
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
Return the priority of a block
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
FILE_PATH_CHANGED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
the block is only in one of the to-do lists
if it is in none then data-node already has it
the block is only in one of the to-do lists
if it is in none then data-node already has it
FILE_PATH_CHANGED
addBlock(DatanodeDescriptor, Block, String)
public void addBlock(DatanodeDescriptor node, Block block, String delHint) throws IOException
// decrement number of blocks scheduled to this datanode.
node.decBlocksScheduled();
// get the deletion hint node
DatanodeDescriptor delHintNode = null;
if (delHint != null && delHint.length() != 0) {
delHintNode = namesystem.getDatanode(delHint);
if (delHintNode == null) {
NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint);
}
}
//
// Modify the blocks->datanode map and node's map.
//
pendingReplications.remove(block);
// blockReceived reports a finalized block
Collection toAdd = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC);
// the block is only in one of the to-do lists
// if it is in none then data-node already has it
assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists.";
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, delHintNode, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
The next two methods test the various cases under which we must conclude
the replica is corrupt, or under construction. These are laid out
as switch statements, on the theory that it is easier to understand
the combinatorics of reportedState and ucState that way. It should be
at least as efficient as boolean expressions.
The next two methods test the various cases under which we must conclude
the replica is corrupt, or under construction. These are laid out
as switch statements, on the theory that it is easier to understand
the combinatorics of reportedState and ucState that way. It should be
at least as efficient as boolean expressions.
switch(reportedState) {
case FINALIZED:
switch(ucState) {
case COMPLETE:
case COMMITTED:
return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes());
default:
return false;
}
case RBW:
case RWR:
return storedBlock.isComplete();
// should not be reported
case RUR:
// should not be reported
case TEMPORARY:
default:
FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes());
return true;
}
// add to under-construction list
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
BlockInfo delimiter = new BlockInfo(new Block(), 1);
boolean added = dn.addBlock(delimiter);
assert added : "Delimiting block cannot be present in the node";
if (newReport == null)
newReport = new BlockListAsLongs();
// scan the report and process newly reported blocks
BlockReportIterator itBR = newReport.getBlockReportIterator();
while (itBR.hasNext()) {
Block iblk = itBR.next();
ReplicaState iState = itBR.getCurrentReplicaState();
BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
if (storedBlock != null && storedBlock.findDatanode(dn) >= 0)
dn.moveBlockToHead(storedBlock);
}
// collect blocks that have not been reported
// all of them are next to the delimiter
Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn);
while (it.hasNext()) toRemove.add(it.next());
dn.removeBlock(delimiter);
public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException
boolean isFirstBlockReport = (node.numBlocks() == 0);
if (isFirstBlockReport) {
// Initial block reports can be processed a lot more efficiently than
// ordinary block reports. This shortens NN restart times.
processFirstBlockReport(node, report);
return;
}
// Normal case:
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
Collection toAdd = new LinkedList();
Collection toRemove = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC);
// Process the blocks on each queue
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (Block b : toRemove) {
removeStoredBlock(b, node);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, null, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
// decrement number of blocks scheduled to this datanode.
node.decBlocksScheduled();
// get the deletion hint node
DatanodeDescriptor delHintNode = null;
if (delHint != null && delHint.length() != 0) {
delHintNode = namesystem.getDatanode(delHint);
if (delHintNode == null) {
NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint);
}
}
//
// Modify the blocks->datanode map and node's map.
//
pendingReplications.remove(block);
// blockReceived reports a finalized block
Collection toAdd = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC);
// the block is only in one of the to-do lists
// if it is in none then data-node already has it
assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists.";
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, delHintNode, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
The next two methods test the various cases under which we must conclude
the replica is corrupt, or under construction. These are laid out
as switch statements, on the theory that it is easier to understand
the combinatorics of reportedState and ucState that way. It should be
at least as efficient as boolean expressions.
The next two methods test the various cases under which we must conclude
the replica is corrupt, or under construction. These are laid out
as switch statements, on the theory that it is easier to understand
the combinatorics of reportedState and ucState that way. It should be
at least as efficient as boolean expressions.
switch(reportedState) {
case FINALIZED:
switch(ucState) {
case COMPLETE:
case COMMITTED:
return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes());
default:
return false;
}
case RBW:
case RWR:
return storedBlock.isComplete();
// should not be reported
case RUR:
// should not be reported
case TEMPORARY:
default:
FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes());
return true;
}
// add to under-construction list
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
BlockInfo delimiter = new BlockInfo(new Block(), 1);
boolean added = dn.addBlock(delimiter);
assert added : "Delimiting block cannot be present in the node";
if (newReport == null)
newReport = new BlockListAsLongs();
// scan the report and process newly reported blocks
BlockReportIterator itBR = newReport.getBlockReportIterator();
while (itBR.hasNext()) {
Block iblk = itBR.next();
ReplicaState iState = itBR.getCurrentReplicaState();
BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
if (storedBlock != null && storedBlock.findDatanode(dn) >= 0)
dn.moveBlockToHead(storedBlock);
}
// collect blocks that have not been reported
// all of them are next to the delimiter
Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn);
while (it.hasNext()) toRemove.add(it.next());
dn.removeBlock(delimiter);
public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException
boolean isFirstBlockReport = (node.numBlocks() == 0);
if (isFirstBlockReport) {
// Initial block reports can be processed a lot more efficiently than
// ordinary block reports. This shortens NN restart times.
processFirstBlockReport(node, report);
return;
}
// Normal case:
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
Collection toAdd = new LinkedList();
Collection toRemove = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC);
// Process the blocks on each queue
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (Block b : toRemove) {
removeStoredBlock(b, node);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, null, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
add a block to a under replication queue according to its priority
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
add a block to a under replication queue according to its priority
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
FILE_PATH_CHANGED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
Return the priority of a block
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
Return the priority of a block
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
FILE_PATH_CHANGED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
Keeps a TreeSet for every named node. Each treeset contains
a list of the blocks that are \"extra\" at that location. We'll
eventually remove these extras.
Mapping: StorageID -> TreeSet
add a block to a under replication queue according to its priority
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
add a block to a under replication queue according to its priority
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
FILE_PATH_CHANGED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
Return the priority of a block
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
Return the priority of a block
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
FILE_PATH_CHANGED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
the block is only in one of the to-do lists
if it is in none then data-node already has it
the block is only in one of the to-do lists
if it is in none then data-node already has it
FILE_PATH_CHANGED
addBlock(DatanodeDescriptor, Block, String)
public void addBlock(DatanodeDescriptor node, Block block, String delHint) throws IOException
// decrement number of blocks scheduled to this datanode.
node.decBlocksScheduled();
// get the deletion hint node
DatanodeDescriptor delHintNode = null;
if (delHint != null && delHint.length() != 0) {
delHintNode = namesystem.getDatanode(delHint);
if (delHintNode == null) {
NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint);
}
}
//
// Modify the blocks->datanode map and node's map.
//
pendingReplications.remove(block);
// blockReceived reports a finalized block
Collection toAdd = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC);
// the block is only in one of the to-do lists
// if it is in none then data-node already has it
assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists.";
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, delHintNode, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
The next two methods test the various cases under which we must conclude
the replica is corrupt, or under construction. These are laid out
as switch statements, on the theory that it is easier to understand
the combinatorics of reportedState and ucState that way. It should be
at least as efficient as boolean expressions.
The next two methods test the various cases under which we must conclude
the replica is corrupt, or under construction. These are laid out
as switch statements, on the theory that it is easier to understand
the combinatorics of reportedState and ucState that way. It should be
at least as efficient as boolean expressions.
switch(reportedState) {
case FINALIZED:
switch(ucState) {
case COMPLETE:
case COMMITTED:
return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes());
default:
return false;
}
case RBW:
case RWR:
return storedBlock.isComplete();
// should not be reported
case RUR:
// should not be reported
case TEMPORARY:
default:
FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes());
return true;
}
// add to under-construction list
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
BlockInfo delimiter = new BlockInfo(new Block(), 1);
boolean added = dn.addBlock(delimiter);
assert added : "Delimiting block cannot be present in the node";
if (newReport == null)
newReport = new BlockListAsLongs();
// scan the report and process newly reported blocks
BlockReportIterator itBR = newReport.getBlockReportIterator();
while (itBR.hasNext()) {
Block iblk = itBR.next();
ReplicaState iState = itBR.getCurrentReplicaState();
BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
if (storedBlock != null && storedBlock.findDatanode(dn) >= 0)
dn.moveBlockToHead(storedBlock);
}
// collect blocks that have not been reported
// all of them are next to the delimiter
Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn);
while (it.hasNext()) toRemove.add(it.next());
dn.removeBlock(delimiter);
public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException
boolean isFirstBlockReport = (node.numBlocks() == 0);
if (isFirstBlockReport) {
// Initial block reports can be processed a lot more efficiently than
// ordinary block reports. This shortens NN restart times.
processFirstBlockReport(node, report);
return;
}
// Normal case:
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
Collection toAdd = new LinkedList();
Collection toRemove = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC);
// Process the blocks on each queue
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (Block b : toRemove) {
removeStoredBlock(b, node);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, null, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
For some reason even with an MBeanException available to them
Runtime exceptionscan still find their way through, so treat them
the same as MBeanException
LOG.debug("Listing beans for " + qry);
Set names = null;
names = mBeanServer.queryNames(qry, null);
jg.writeArrayFieldStart("beans");
Iterator it = names.iterator();
while (it.hasNext()) {
ObjectName oname = it.next();
MBeanInfo minfo;
String code = "";
Object attributeinfo = null;
try {
minfo = mBeanServer.getMBeanInfo(oname);
code = minfo.getClassName();
String prs = "";
try {
if ("org.apache.commons.modeler.BaseModelMBean".equals(code)) {
prs = "modelerType";
code = (String) mBeanServer.getAttribute(oname, prs);
}
if (attribute != null) {
prs = attribute;
attributeinfo = mBeanServer.getAttribute(oname, prs);
}
} catch (AttributeNotFoundException e) {
// If the modelerType attribute was not found, the class name is used
// instead.
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
} catch (MBeanException e) {
// The code inside the attribute getter threw an exception so log it,
// and fall back on the class name
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
} catch (RuntimeException e) {
// For some reason even with an MBeanException available to them
// Runtime exceptionscan still find their way through, so treat them
// the same as MBeanException
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
} catch (ReflectionException e) {
// This happens when the code inside the JMX bean (setter?? from the
// java docs) threw an exception, so log it and fall back on the
// class name
LOG.error("getting attribute " + prs + " of " + oname + " threw an exception", e);
}
} catch (InstanceNotFoundException e) {
// Ignored for some reason the bean was not found so don't output it
continue;
} catch (IntrospectionException e) {
// This is an internal error, something odd happened with reflection so
// log it and don't output the bean.
LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e);
continue;
} catch (ReflectionException e) {
// This happens when the code inside the JMX bean threw an exception, so
// log it and don't output the bean.
LOG.error("Problem while trying to process JMX query: " + qry + " with MBean " + oname, e);
continue;
}
jg.writeStartObject();
jg.writeStringField("name", oname.toString());
jg.writeStringField("modelerType", code);
if ((attribute != null) && (attributeinfo == null)) {
jg.writeStringField("result", "ERROR");
jg.writeStringField("message", "No attribute with name " + attribute + " was found.");
jg.writeEndObject();
jg.writeEndArray();
jg.close();
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (attribute != null) {
writeAttribute(jg, attribute, attributeinfo);
} else {
MBeanAttributeInfo[] attrs = minfo.getAttributes();
for (int i = 0; i < attrs.length; i++) {
writeAttribute(jg, oname, attrs[i]);
}
}
jg.writeEndObject();
}
jg.writeEndArray();
if (!(child instanceof MachineNode)) {
throw new IllegalArgumentException("Only MachineNode can be added to RackNode");
}
return super.addChild(child);
Since distributed cache is unmodified in dfs
between two job runs, it should not be present more than once
in any of the task tracker, in which job ran.
SATD_ADDED
testDistributedCache()
public void testDistributedCache() throws Exception
Configuration conf = new Configuration(cluster.getConf());
JTProtocol wovenClient = cluster.getJTClient().getProxy();
// This counter will check for count of a loop,
// which might become infinite.
int count = 0;
// This boolean will decide whether to run job again
boolean continueLoop = true;
// counter for job Loop
int countLoop = 0;
// This counter incerases with all the tasktrackers in which tasks ran
int taskTrackerCounter = 0;
// This will store all the tasktrackers in which tasks ran
ArrayList taskTrackerCollection = new ArrayList();
do {
SleepJob job = new SleepJob();
job.setConf(conf);
Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100);
DistributedCache.createSymlink(conf);
URI uri = URI.create(uriPath);
DistributedCache.addCacheFile(uri, conf);
JobConf jconf = new JobConf(conf);
// Controls the job till all verification is done
FinishTaskControlAction.configureControlActionForJob(conf);
// Submitting the job
slpJob.submit();
RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));
// counter for job Loop
countLoop++;
TTClient tClient = null;
JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
LOG.info("jInfo is :" + jInfo);
// Assert if jobInfo is null
Assert.assertNotNull("jobInfo is null", jInfo);
// Wait for the job to start running.
count = 0;
while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
UtilsForTests.waitFor(10000);
count++;
jInfo = wovenClient.getJobInfo(rJob.getID());
// If the count goes beyond a point, then break; This is to avoid
// infinite loop under unforeseen circumstances. Testcase will anyway
// fail later.
if (count > 10) {
Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point");
}
}
LOG.info("job id is :" + rJob.getID().toString());
TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID());
boolean distCacheFileIsFound;
for (TaskInfo taskInfo : taskInfos) {
distCacheFileIsFound = false;
String[] taskTrackers = taskInfo.getTaskTrackers();
for (String taskTracker : taskTrackers) {
// Formatting tasktracker to get just its FQDN
taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
LOG.info("taskTracker is :" + taskTracker);
// This will be entered from the second job onwards
if (countLoop > 1) {
if (taskTracker != null) {
continueLoop = taskTrackerCollection.contains(taskTracker);
}
if (!continueLoop) {
break;
}
}
// Collecting the tasktrackers
if (taskTracker != null)
taskTrackerCollection.add(taskTracker);
// we have loopped through enough number of times to look for task
// getting submitted on same tasktrackers.The same tasktracker
// for subsequent jobs was not hit maybe because of many number
// of tasktrackers. So, testcase has to stop here.
if (countLoop > 2) {
continueLoop = false;
}
tClient = cluster.getTTClient(taskTracker);
// tClient maybe null because the task is already dead. Ex: setup
if (tClient == null) {
continue;
}
String[] localDirs = tClient.getMapredLocalDirs();
int distributedFileCount = 0;
// Go to every single path
for (String localDir : localDirs) {
// Public Distributed cache will always be stored under
// mapre.local.dir/tasktracker/archive
localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir();
LOG.info("localDir is : " + localDir);
// Get file status of all the directories
// and files under that path.
FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true);
for (FileStatus fileStatus : fileStatuses) {
Path path = fileStatus.getPath();
LOG.info("path is :" + path.toString());
// Checking if the received path ends with
// the distributed filename
distCacheFileIsFound = (path.toString()).endsWith(distributedFileName);
// If file is found, check for its permission.
// Since the file is found break out of loop
if (distCacheFileIsFound) {
LOG.info("PATH found is :" + path.toString());
distributedFileCount++;
String filename = path.getName();
FsPermission fsPerm = fileStatus.getPermission();
Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777")));
}
}
}
// Since distributed cache is unmodified in dfs
// between two job runs, it should not be present more than once
// in any of the task tracker, in which job ran.
if (distributedFileCount > 1) {
Assert.fail("The distributed cache file is more than one");
} else if (distributedFileCount < 1)
Assert.fail("The distributed cache file is less than one");
if (!distCacheFileIsFound) {
Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false);
}
}
}
// Allow the job to continue through MR control job.
for (TaskInfo taskInfoRemaining : taskInfos) {
FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID()));
Collection tts = cluster.getTTClients();
for (TTClient cli : tts) {
cli.getProxy().sendAction(action);
}
}
// Killing the job because all the verification needed
// for this testcase is completed.
rJob.killJob();
} while (continueLoop);
tClient maybe null because the task is already dead. Ex: setup
SATD_ADDED
testDistributedCache()
public void testDistributedCache() throws Exception
Configuration conf = new Configuration(cluster.getConf());
JTProtocol wovenClient = cluster.getJTClient().getProxy();
// This counter will check for count of a loop,
// which might become infinite.
int count = 0;
// This boolean will decide whether to run job again
boolean continueLoop = true;
// counter for job Loop
int countLoop = 0;
// This counter incerases with all the tasktrackers in which tasks ran
int taskTrackerCounter = 0;
// This will store all the tasktrackers in which tasks ran
ArrayList taskTrackerCollection = new ArrayList();
do {
SleepJob job = new SleepJob();
job.setConf(conf);
Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100);
DistributedCache.createSymlink(conf);
URI uri = URI.create(uriPath);
DistributedCache.addCacheFile(uri, conf);
JobConf jconf = new JobConf(conf);
// Controls the job till all verification is done
FinishTaskControlAction.configureControlActionForJob(conf);
// Submitting the job
slpJob.submit();
RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));
// counter for job Loop
countLoop++;
TTClient tClient = null;
JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
LOG.info("jInfo is :" + jInfo);
// Assert if jobInfo is null
Assert.assertNotNull("jobInfo is null", jInfo);
// Wait for the job to start running.
count = 0;
while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
UtilsForTests.waitFor(10000);
count++;
jInfo = wovenClient.getJobInfo(rJob.getID());
// If the count goes beyond a point, then break; This is to avoid
// infinite loop under unforeseen circumstances. Testcase will anyway
// fail later.
if (count > 10) {
Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point");
}
}
LOG.info("job id is :" + rJob.getID().toString());
TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID());
boolean distCacheFileIsFound;
for (TaskInfo taskInfo : taskInfos) {
distCacheFileIsFound = false;
String[] taskTrackers = taskInfo.getTaskTrackers();
for (String taskTracker : taskTrackers) {
// Formatting tasktracker to get just its FQDN
taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
LOG.info("taskTracker is :" + taskTracker);
// This will be entered from the second job onwards
if (countLoop > 1) {
if (taskTracker != null) {
continueLoop = taskTrackerCollection.contains(taskTracker);
}
if (!continueLoop) {
break;
}
}
// Collecting the tasktrackers
if (taskTracker != null)
taskTrackerCollection.add(taskTracker);
// we have loopped through enough number of times to look for task
// getting submitted on same tasktrackers.The same tasktracker
// for subsequent jobs was not hit maybe because of many number
// of tasktrackers. So, testcase has to stop here.
if (countLoop > 2) {
continueLoop = false;
}
tClient = cluster.getTTClient(taskTracker);
// tClient maybe null because the task is already dead. Ex: setup
if (tClient == null) {
continue;
}
String[] localDirs = tClient.getMapredLocalDirs();
int distributedFileCount = 0;
// Go to every single path
for (String localDir : localDirs) {
// Public Distributed cache will always be stored under
// mapre.local.dir/tasktracker/archive
localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir();
LOG.info("localDir is : " + localDir);
// Get file status of all the directories
// and files under that path.
FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true);
for (FileStatus fileStatus : fileStatuses) {
Path path = fileStatus.getPath();
LOG.info("path is :" + path.toString());
// Checking if the received path ends with
// the distributed filename
distCacheFileIsFound = (path.toString()).endsWith(distributedFileName);
// If file is found, check for its permission.
// Since the file is found break out of loop
if (distCacheFileIsFound) {
LOG.info("PATH found is :" + path.toString());
distributedFileCount++;
String filename = path.getName();
FsPermission fsPerm = fileStatus.getPermission();
Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777")));
}
}
}
// Since distributed cache is unmodified in dfs
// between two job runs, it should not be present more than once
// in any of the task tracker, in which job ran.
if (distributedFileCount > 1) {
Assert.fail("The distributed cache file is more than one");
} else if (distributedFileCount < 1)
Assert.fail("The distributed cache file is less than one");
if (!distCacheFileIsFound) {
Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false);
}
}
}
// Allow the job to continue through MR control job.
for (TaskInfo taskInfoRemaining : taskInfos) {
FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID()));
Collection tts = cluster.getTTClients();
for (TTClient cli : tts) {
cli.getProxy().sendAction(action);
}
}
// Killing the job because all the verification needed
// for this testcase is completed.
rJob.killJob();
} while (continueLoop);
TCP_NODELAY is crucial here because of bad interactions between
Nagle's Algorithm and Delayed ACKs. With connection keepalive
between the client and DN, the conversation looks like:
1. Client -> DN: Read block X
2. DN -> Client: data for block X
3. Client -> DN: Status OK (successful read)
4. Client -> DN: Read block Y
The fact that step #3 and #4 are both in the client->DN direction
triggers Nagling. If the DN is using delayed ACKs, this results
in a delay of 40ms or more.
TCP_NODELAY disables nagling and thus avoids this performance
disaster.
Finish the map task on the tracker 1. Finishing it here to work
around bug in the FakeJobInProgress object
SATD_ADDED
testReservationOnBlacklistedTracker()
public void testReservationOnBlacklistedTracker() throws Exception
TaskAttemptID[] taskAttemptID = new TaskAttemptID[3];
JobConf conf = new JobConf();
conf.setSpeculativeExecution(false);
conf.setNumMapTasks(2);
conf.setNumReduceTasks(2);
conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70");
conf.set(JobContext.MAP_FAILURES_MAX_PERCENT, ".70");
conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false);
conf.setMaxTaskFailuresPerTracker(1);
FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
job.setClusterSize(trackers.length);
job.initTasks();
TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]);
TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
TaskTracker tt3 = jobTracker.getTaskTracker(trackers[2]);
TaskTrackerStatus status1 = new TaskTrackerStatus(trackers[0], JobInProgress.convertTrackerNameToHostName(trackers[0]), 0, new ArrayList(), 0, 2, 2);
TaskTrackerStatus status2 = new TaskTrackerStatus(trackers[1], JobInProgress.convertTrackerNameToHostName(trackers[1]), 0, new ArrayList(), 0, 2, 2);
TaskTrackerStatus status3 = new TaskTrackerStatus(trackers[1], JobInProgress.convertTrackerNameToHostName(trackers[1]), 0, new ArrayList(), 0, 2, 2);
tt1.setStatus(status1);
tt2.setStatus(status2);
tt3.setStatus(status3);
tt1.reserveSlots(TaskType.MAP, job, 2);
tt1.reserveSlots(TaskType.REDUCE, job, 2);
tt3.reserveSlots(TaskType.MAP, job, 2);
tt3.reserveSlots(TaskType.REDUCE, job, 2);
assertEquals("Trackers not reserved for the job : maps", 2, job.getNumReservedTaskTrackersForMaps());
assertEquals("Trackers not reserved for the job : reduces", 2, job.getNumReservedTaskTrackersForReduces());
ClusterMetrics metrics = jobTracker.getClusterMetrics();
assertEquals("reserved map slots do not match", 4, metrics.getReservedMapSlots());
assertEquals("reserved reduce slots do not match", 4, metrics.getReservedReduceSlots());
/*
* FakeJobInProgress.findMapTask does not handle
* task failures. So working around it by failing
* reduce and blacklisting tracker.
* Then finish the map task later.
*/
TaskAttemptID mTid = job.findMapTask(trackers[0]);
TaskAttemptID rTid = job.findReduceTask(trackers[0]);
// Task should blacklist the tasktracker.
job.failTask(rTid);
assertEquals("Tracker 0 not blacklisted for the job", 1, job.getBlackListedTrackers().size());
assertEquals("Extra Trackers reserved for the job : maps", 1, job.getNumReservedTaskTrackersForMaps());
assertEquals("Extra Trackers reserved for the job : reduces", 1, job.getNumReservedTaskTrackersForReduces());
metrics = jobTracker.getClusterMetrics();
assertEquals("reserved map slots do not match", 2, metrics.getReservedMapSlots());
assertEquals("reserved reduce slots do not match", 2, metrics.getReservedReduceSlots());
// Finish the map task on the tracker 1. Finishing it here to work
// around bug in the FakeJobInProgress object
job.finishTask(mTid);
mTid = job.findMapTask(trackers[1]);
rTid = job.findReduceTask(trackers[1]);
job.finishTask(mTid);
job.finishTask(rTid);
rTid = job.findReduceTask(trackers[1]);
job.finishTask(rTid);
assertEquals("Job didnt complete successfully complete", job.getStatus().getRunState(), JobStatus.SUCCEEDED);
assertEquals("Trackers not unreserved for the job : maps", 0, job.getNumReservedTaskTrackersForMaps());
assertEquals("Trackers not unreserved for the job : reduces", 0, job.getNumReservedTaskTrackersForReduces());
metrics = jobTracker.getClusterMetrics();
assertEquals("reserved map slots do not match", 0, metrics.getReservedMapSlots());
assertEquals("reserved reduce slots do not match", 0, metrics.getReservedReduceSlots());
solve();
FileSplit[] result = new FileSplit[realSplits.length];
int left = 0;
int right = realSplits.length - 1;
for (int i = 0; i < splits.length; ++i) {
if (splits[i].isAssigned) {
// copy the split and fix up the locations
((TeraFileSplit) realSplits[i]).setLocations(new String[] { splits[i].locations.get(0).hostname });
result[left++] = realSplits[i];
} else {
result[right--] = realSplits[i];
}
}
List ret = new ArrayList();
for (FileSplit fs : result) {
ret.add(fs);
}
return ret;
Don't assign reduce tasks to the hilt!
Leave some free slots in the cluster for future task-failures,
speculative tasks etc. beyond the highest priority job
SATD_ADDED
assignTasks(TaskTracker)
public synchronized List assignTasks(TaskTracker taskTracker) throws IOException
TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
final int numTaskTrackers = clusterStatus.getTaskTrackers();
final int clusterMapCapacity = clusterStatus.getMaxMapTasks();
final int clusterReduceCapacity = clusterStatus.getMaxReduceTasks();
Collection jobQueue = jobQueueJobInProgressListener.getJobQueue();
//
// Get map + reduce counts for the current tracker.
//
final int trackerMapCapacity = taskTrackerStatus.getMaxMapSlots();
final int trackerReduceCapacity = taskTrackerStatus.getMaxReduceSlots();
final int trackerRunningMaps = taskTrackerStatus.countMapTasks();
final int trackerRunningReduces = taskTrackerStatus.countReduceTasks();
// Assigned tasks
List assignedTasks = new ArrayList();
//
// Compute (running + pending) map and reduce task numbers across pool
//
int remainingReduceLoad = 0;
int remainingMapLoad = 0;
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() == JobStatus.RUNNING) {
remainingMapLoad += (job.desiredMaps() - job.finishedMaps());
if (job.scheduleReduces()) {
remainingReduceLoad += (job.desiredReduces() - job.finishedReduces());
}
}
}
}
// Compute the 'load factor' for maps and reduces
double mapLoadFactor = 0.0;
if (clusterMapCapacity > 0) {
mapLoadFactor = (double) remainingMapLoad / clusterMapCapacity;
}
double reduceLoadFactor = 0.0;
if (clusterReduceCapacity > 0) {
reduceLoadFactor = (double) remainingReduceLoad / clusterReduceCapacity;
}
//
// In the below steps, we allocate first map tasks (if appropriate),
// and then reduce tasks if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We assign tasks to the current taskTracker if the given machine
// has a workload that's less than the maximum load of that kind of
// task.
// However, if the cluster is close to getting loaded i.e. we don't
// have enough _padding_ for speculative executions etc., we only
// schedule the "highest priority" task i.e. the task from the job
// with the highest priority.
//
final int trackerCurrentMapCapacity = Math.min((int) Math.ceil(mapLoadFactor * trackerMapCapacity), trackerMapCapacity);
int availableMapSlots = trackerCurrentMapCapacity - trackerRunningMaps;
boolean exceededMapPadding = false;
if (availableMapSlots > 0) {
exceededMapPadding = exceededPadding(true, clusterStatus, trackerMapCapacity);
}
int numLocalMaps = 0;
int numNonLocalMaps = 0;
scheduleMaps: for (int i = 0; i < availableMapSlots; ++i) {
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = null;
// Try to schedule a node-local or rack-local Map task
t = job.obtainNewLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numLocalMaps;
// Don't assign map tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededMapPadding) {
break scheduleMaps;
}
// Try all jobs again for the next Map task
break;
}
// Try to schedule a node-local or rack-local Map task
t = job.obtainNewNonLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numNonLocalMaps;
// We assign at most 1 off-switch or speculative task
// This is to prevent TaskTrackers from stealing local-tasks
// from other TaskTrackers.
break scheduleMaps;
}
}
}
}
int assignedMaps = assignedTasks.size();
//
// Same thing, but for reduce tasks
// However we _never_ assign more than 1 reduce task per heartbeat
//
final int trackerCurrentReduceCapacity = Math.min((int) Math.ceil(reduceLoadFactor * trackerReduceCapacity), trackerReduceCapacity);
final int availableReduceSlots = Math.min((trackerCurrentReduceCapacity - trackerRunningReduces), 1);
boolean exceededReducePadding = false;
if (availableReduceSlots > 0) {
exceededReducePadding = exceededPadding(false, clusterStatus, trackerReduceCapacity);
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() != JobStatus.RUNNING || job.numReduceTasks == 0) {
continue;
}
Task t = job.obtainNewReduceTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
break;
}
// Don't assign reduce tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededReducePadding) {
break;
}
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Task assignments for " + taskTrackerStatus.getTrackerName() + " --> " + "[" + mapLoadFactor + ", " + trackerMapCapacity + ", " + trackerCurrentMapCapacity + ", " + trackerRunningMaps + "] -> [" + (trackerCurrentMapCapacity - trackerRunningMaps) + ", " + assignedMaps + " (" + numLocalMaps + ", " + numNonLocalMaps + ")] [" + reduceLoadFactor + ", " + trackerReduceCapacity + ", " + trackerCurrentReduceCapacity + "," + trackerRunningReduces + "] -> [" + (trackerCurrentReduceCapacity - trackerRunningReduces) + ", " + (assignedTasks.size() - assignedMaps) + "]");
}
return assignedTasks;
Don't assign map tasks to the hilt!
Leave some free slots in the cluster for future task-failures,
speculative tasks etc. beyond the highest priority job
SATD_ADDED
assignTasks(TaskTracker)
public synchronized List assignTasks(TaskTracker taskTracker) throws IOException
TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
final int numTaskTrackers = clusterStatus.getTaskTrackers();
final int clusterMapCapacity = clusterStatus.getMaxMapTasks();
final int clusterReduceCapacity = clusterStatus.getMaxReduceTasks();
Collection jobQueue = jobQueueJobInProgressListener.getJobQueue();
//
// Get map + reduce counts for the current tracker.
//
final int trackerMapCapacity = taskTrackerStatus.getMaxMapSlots();
final int trackerReduceCapacity = taskTrackerStatus.getMaxReduceSlots();
final int trackerRunningMaps = taskTrackerStatus.countMapTasks();
final int trackerRunningReduces = taskTrackerStatus.countReduceTasks();
// Assigned tasks
List assignedTasks = new ArrayList();
//
// Compute (running + pending) map and reduce task numbers across pool
//
int remainingReduceLoad = 0;
int remainingMapLoad = 0;
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() == JobStatus.RUNNING) {
remainingMapLoad += (job.desiredMaps() - job.finishedMaps());
if (job.scheduleReduces()) {
remainingReduceLoad += (job.desiredReduces() - job.finishedReduces());
}
}
}
}
// Compute the 'load factor' for maps and reduces
double mapLoadFactor = 0.0;
if (clusterMapCapacity > 0) {
mapLoadFactor = (double) remainingMapLoad / clusterMapCapacity;
}
double reduceLoadFactor = 0.0;
if (clusterReduceCapacity > 0) {
reduceLoadFactor = (double) remainingReduceLoad / clusterReduceCapacity;
}
//
// In the below steps, we allocate first map tasks (if appropriate),
// and then reduce tasks if appropriate. We go through all jobs
// in order of job arrival; jobs only get serviced if their
// predecessors are serviced, too.
//
//
// We assign tasks to the current taskTracker if the given machine
// has a workload that's less than the maximum load of that kind of
// task.
// However, if the cluster is close to getting loaded i.e. we don't
// have enough _padding_ for speculative executions etc., we only
// schedule the "highest priority" task i.e. the task from the job
// with the highest priority.
//
final int trackerCurrentMapCapacity = Math.min((int) Math.ceil(mapLoadFactor * trackerMapCapacity), trackerMapCapacity);
int availableMapSlots = trackerCurrentMapCapacity - trackerRunningMaps;
boolean exceededMapPadding = false;
if (availableMapSlots > 0) {
exceededMapPadding = exceededPadding(true, clusterStatus, trackerMapCapacity);
}
int numLocalMaps = 0;
int numNonLocalMaps = 0;
scheduleMaps: for (int i = 0; i < availableMapSlots; ++i) {
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
Task t = null;
// Try to schedule a node-local or rack-local Map task
t = job.obtainNewLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numLocalMaps;
// Don't assign map tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededMapPadding) {
break scheduleMaps;
}
// Try all jobs again for the next Map task
break;
}
// Try to schedule a node-local or rack-local Map task
t = job.obtainNewNonLocalMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
++numNonLocalMaps;
// We assign at most 1 off-switch or speculative task
// This is to prevent TaskTrackers from stealing local-tasks
// from other TaskTrackers.
break scheduleMaps;
}
}
}
}
int assignedMaps = assignedTasks.size();
//
// Same thing, but for reduce tasks
// However we _never_ assign more than 1 reduce task per heartbeat
//
final int trackerCurrentReduceCapacity = Math.min((int) Math.ceil(reduceLoadFactor * trackerReduceCapacity), trackerReduceCapacity);
final int availableReduceSlots = Math.min((trackerCurrentReduceCapacity - trackerRunningReduces), 1);
boolean exceededReducePadding = false;
if (availableReduceSlots > 0) {
exceededReducePadding = exceededPadding(false, clusterStatus, trackerReduceCapacity);
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
if (job.getStatus().getRunState() != JobStatus.RUNNING || job.numReduceTasks == 0) {
continue;
}
Task t = job.obtainNewReduceTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (t != null) {
assignedTasks.add(t);
break;
}
// Don't assign reduce tasks to the hilt!
// Leave some free slots in the cluster for future task-failures,
// speculative tasks etc. beyond the highest priority job
if (exceededReducePadding) {
break;
}
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Task assignments for " + taskTrackerStatus.getTrackerName() + " --> " + "[" + mapLoadFactor + ", " + trackerMapCapacity + ", " + trackerCurrentMapCapacity + ", " + trackerRunningMaps + "] -> [" + (trackerCurrentMapCapacity - trackerRunningMaps) + ", " + assignedMaps + " (" + numLocalMaps + ", " + numNonLocalMaps + ")] [" + reduceLoadFactor + ", " + trackerReduceCapacity + ", " + trackerCurrentReduceCapacity + "," + trackerRunningReduces + "] -> [" + (trackerCurrentReduceCapacity - trackerRunningReduces) + ", " + (assignedTasks.size() - assignedMaps) + "]");
}
return assignedTasks;
if (offset == args.length) {
throw new IllegalArgumentException(" not specified in " + cmd);
}
long n = StringUtils.TraditionalBinaryPrefix.string2long(args[offset]);
if (n <= 0) {
throw new IllegalArgumentException("n = " + n + " <= 0 in " + cmd);
}
return n;
TODO: when modification times can be set, directories should be
emitted to reducers so they might be preserved. Also, mkdirs does
SATD_ADDED
parseLong(String[], int)
private long parseLong(String[] args, int offset)
if (offset == args.length) {
throw new IllegalArgumentException(" not specified in " + cmd);
}
long n = StringUtils.TraditionalBinaryPrefix.string2long(args[offset]);
if (n <= 0) {
throw new IllegalArgumentException("n = " + n + " <= 0 in " + cmd);
}
return n;
TextInputFormat format = new TextInputFormat();
format.configure(jConf);
// work around 256-byte/22-splits issue
format.setMinSplitSize(5500);
// here's Nth pair of DecompressorStreams:
InputSplit[] splits = format.getSplits(jConf, 100);
assertEquals("compressed splits == 2", 2, splits.length);
FileSplit tmp = (FileSplit) splits[0];
if (tmp.getPath().getName().equals("testCompressThenConcat.txt.gz")) {
System.out.println(" (swapping)");
splits[0] = splits[1];
splits[1] = tmp;
}
// testConcatThenCompress (single)
List results = readSplit(format, splits[0], jConf);
assertEquals("splits[0] length (num lines)", 84, results.size());
assertEquals("splits[0][0]", "Call me Ishmael. Some years ago--never mind how long precisely--having", results.get(0).toString());
assertEquals("splits[0][42]", "Tell me, does the magnetic virtue of the needles of the compasses of", results.get(42).toString());
// testCompressThenConcat (multi)
results = readSplit(format, splits[1], jConf);
assertEquals("splits[1] length (num lines)", 84, results.size());
assertEquals("splits[1][0]", "Call me Ishmael. Some years ago--never mind how long precisely--having", results.get(0).toString());
assertEquals("splits[1][42]", "Tell me, does the magnetic virtue of the needles of the compasses of", results.get(42).toString());
ideally would add some offsets/shifts in here (e.g., via extra header
data?), but...significant work to hand-generate each header, and no
bzip2 spec for reference
test GzipZlibDecompressor (native), just to be sure
(FIXME? could move this call to testGzip(), but would need filename
setup above) (alternatively, maybe just nuke testGzip() and extend this?)
SATD_ADDED
testBuiltInGzipDecompressor()
public void testBuiltInGzipDecompressor() throws IOException
JobConf jobConf = new JobConf(defaultConf);
jobConf.setBoolean("io.native.lib.available", false);
CompressionCodec gzip = new GzipCodec();
ReflectionUtils.setConf(gzip, jobConf);
localFs.delete(workDir, true);
assertEquals("[non-native (Java) codec]", org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor.class, gzip.getDecompressorType());
System.out.println(COLOR_BR_YELLOW + "testBuiltInGzipDecompressor() using" + " non-native (Java Inflater) Decompressor (" + gzip.getDecompressorType() + ")" + COLOR_NORMAL);
// copy single-member test file to HDFS
String fn1 = "testConcatThenCompress.txt" + gzip.getDefaultExtension();
Path fnLocal1 = new Path(System.getProperty("test.concat.data", "/tmp"), fn1);
Path fnHDFS1 = new Path(workDir, fn1);
localFs.copyFromLocalFile(fnLocal1, fnHDFS1);
// copy multiple-member test file to HDFS
// (actually in "seekable gzip" format, a la JIRA PIG-42)
String fn2 = "testCompressThenConcat.txt" + gzip.getDefaultExtension();
Path fnLocal2 = new Path(System.getProperty("test.concat.data", "/tmp"), fn2);
Path fnHDFS2 = new Path(workDir, fn2);
localFs.copyFromLocalFile(fnLocal2, fnHDFS2);
FileInputFormat.setInputPaths(jobConf, workDir);
// here's first pair of DecompressorStreams:
final FileInputStream in1 = new FileInputStream(fnLocal1.toString());
final FileInputStream in2 = new FileInputStream(fnLocal2.toString());
assertEquals("concat bytes available", 2734, in1.available());
// w/hdr CRC
assertEquals("concat bytes available", 3413, in2.available());
CompressionInputStream cin2 = gzip.createInputStream(in2);
LineReader in = new LineReader(cin2);
Text out = new Text();
int numBytes, totalBytes = 0, lineNum = 0;
while ((numBytes = in.readLine(out)) > 0) {
++lineNum;
totalBytes += numBytes;
}
in.close();
assertEquals("total uncompressed bytes in concatenated test file", 5346, totalBytes);
assertEquals("total uncompressed lines in concatenated test file", 84, lineNum);
// test BuiltInGzipDecompressor with lots of different input-buffer sizes
doMultipleGzipBufferSizes(jobConf, false);
// test GzipZlibDecompressor (native), just to be sure
// (FIXME? could move this call to testGzip(), but would need filename
// setup above) (alternatively, maybe just nuke testGzip() and extend this?)
doMultipleGzipBufferSizes(jobConf, true);
TODO Write resources version no too
long #1
long #2
long #3
long #4
SATD_ADDED
write(DataOutput)
public void write(DataOutput out) throws IOException
// TODO Write resources version no too
// long #1
WritableUtils.writeVLong(out, cumulativeCpuUsage);
// long #2
WritableUtils.writeVLong(out, virtualMemoryUsage);
// long #3
WritableUtils.writeVLong(out, physicalMemoryUsage);
// long #4
WritableUtils.writeVLong(out, heapUsage);
Do a small regular read. Very likely this will leave unread
data on the socket and make the socket uncacheable.
SATD_ADDED
run()
public void run()
for (int i = 0; i < N_ITERATIONS; ++i) {
int startOff = rand.nextInt((int) fileSize);
int len = 0;
try {
double p = rand.nextDouble();
if (p < PROPORTION_NON_POSITIONAL_READ) {
// Do a small regular read. Very likely this will leave unread
// data on the socket and make the socket uncacheable.
len = Math.min(rand.nextInt(64), (int) fileSize - startOff);
read(startOff, len);
bytesRead += len;
} else {
// Do a positional read most of the time.
len = rand.nextInt((int) (fileSize - startOff));
pRead(startOff, len);
bytesRead += len;
}
} catch (Exception ex) {
LOG.error(getName() + ": Error while testing read at " + startOff + " length " + len);
error = true;
fail(ex.getMessage());
}
}
MiniDFSCluster cluster = null;
int numDataNodes = 2;
short replFactor = 2;
Random random = new Random();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
util.createFiles(fs, "/srcdat", replFactor);
util.waitReplication(fs, "/srcdat", (short) 2);
// Now deliberately remove/truncate meta blocks from the first
// directory of the first datanode. The complete absense of a meta
// file disallows this Datanode to send data to another datanode.
// However, a client is alowed access to this block.
//
File storageDir = MiniDFSCluster.getStorageDir(0, 1);
String bpid = cluster.getNamesystem().getBlockPoolId();
File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
assertTrue("data directory does not exist", data_dir.exists());
File[] blocks = data_dir.listFiles();
assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
int num = 0;
for (int idx = 0; idx < blocks.length; idx++) {
if (blocks[idx].getName().startsWith("blk_") && blocks[idx].getName().endsWith(".meta")) {
num++;
if (num % 3 == 0) {
//
// remove .meta file
//
System.out.println("Deliberately removing file " + blocks[idx].getName());
assertTrue("Cannot remove file.", blocks[idx].delete());
} else if (num % 3 == 1) {
//
// shorten .meta file
//
RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
FileChannel channel = file.getChannel();
int newsize = random.nextInt((int) channel.size() / 2);
System.out.println("Deliberately truncating file " + blocks[idx].getName() + " to size " + newsize + " bytes.");
channel.truncate(newsize);
file.close();
} else {
//
// corrupt a few bytes of the metafile
//
RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
FileChannel channel = file.getChannel();
long position = 0;
//
// The very first time, corrupt the meta header at offset 0
//
if (num != 2) {
position = (long) random.nextInt((int) channel.size());
}
int length = random.nextInt((int) (channel.size() - position + 1));
byte[] buffer = new byte[length];
random.nextBytes(buffer);
channel.write(ByteBuffer.wrap(buffer), position);
System.out.println("Deliberately corrupting file " + blocks[idx].getName() + " at offset " + position + " length " + length);
file.close();
}
}
}
//
// Now deliberately corrupt all meta blocks from the second
// directory of the first datanode
//
storageDir = MiniDFSCluster.getStorageDir(0, 1);
data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
assertTrue("data directory does not exist", data_dir.exists());
blocks = data_dir.listFiles();
assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
int count = 0;
File previous = null;
for (int idx = 0; idx < blocks.length; idx++) {
if (blocks[idx].getName().startsWith("blk_") && blocks[idx].getName().endsWith(".meta")) {
//
// Move the previous metafile into the current one.
//
count++;
if (count % 2 == 0) {
System.out.println("Deliberately insertimg bad crc into files " + blocks[idx].getName() + " " + previous.getName());
assertTrue("Cannot remove file.", blocks[idx].delete());
assertTrue("Cannot corrupt meta file.", previous.renameTo(blocks[idx]));
assertTrue("Cannot recreate empty meta file.", previous.createNewFile());
previous = null;
} else {
previous = blocks[idx];
}
}
}
//
// Only one replica is possibly corrupted. The other replica should still
// be good. Verify.
//
assertTrue("Corrupted replicas not handled properly.", util.checkFiles(fs, "/srcdat"));
System.out.println("All File still have a valid replica");
//
// set replication factor back to 1. This causes only one replica of
// of each block to remain in HDFS. The check is to make sure that
// the corrupted replica generated above is the one that gets deleted.
// This test is currently disabled until HADOOP-1557 is solved.
//
util.setReplication(fs, "/srcdat", (short) 1);
// util.waitReplication(fs, "/srcdat", (short)1);
// System.out.println("All Files done with removing replicas");
// assertTrue("Excess replicas deleted. Corrupted replicas found.",
// util.checkFiles(fs, "/srcdat"));
System.out.println("The excess-corrupted-replica test is disabled " + " pending HADOOP-1557");
util.cleanup(fs, "/srcdat");
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
// set the starting generation for each shard
// when a reduce task fails, a new reduce task
// has to know where to re-start
setShardGeneration(conf, shards);
// iconf.set sets properties in conf
IndexUpdateConfiguration iconf = new IndexUpdateConfiguration(conf);
Shard.setIndexShards(iconf, shards);
// MapTask.MapOutputBuffer uses JobContext.IO_SORT_MB to decide its max buffer size
// (max buffer size = 1/2 * JobContext.IO_SORT_MB).
// Here we half-en JobContext.IO_SORT_MB because we use the other half memory to
// build an intermediate form/index in Combiner.
iconf.setIOSortMB(iconf.getIOSortMB() / 2);
// create the job configuration
JobConf jobConf = new JobConf(conf, IndexUpdater.class);
jobConf.setJobName(this.getClass().getName() + "_" + System.currentTimeMillis());
// provided by application
FileInputFormat.setInputPaths(jobConf, inputPaths);
FileOutputFormat.setOutputPath(jobConf, outputPath);
jobConf.setNumMapTasks(numMapTasks);
// already set shards
jobConf.setNumReduceTasks(shards.length);
jobConf.setInputFormat(iconf.getIndexInputFormatClass());
Path[] inputs = FileInputFormat.getInputPaths(jobConf);
StringBuilder buffer = new StringBuilder(inputs[0].toString());
for (int i = 1; i < inputs.length; i++) {
buffer.append(",");
buffer.append(inputs[i].toString());
}
LOG.info("mapred.input.dir = " + buffer.toString());
LOG.info("mapreduce.output.fileoutputformat.outputdir = " + FileOutputFormat.getOutputPath(jobConf).toString());
LOG.info("mapreduce.job.maps = " + jobConf.getNumMapTasks());
LOG.info("mapreduce.job.reduces = " + jobConf.getNumReduceTasks());
LOG.info(shards.length + " shards = " + iconf.getIndexShards());
// better if we don't create the input format instance
LOG.info("mapred.input.format.class = " + jobConf.getInputFormat().getClass().getName());
// set by the system
jobConf.setMapOutputKeyClass(IndexUpdateMapper.getMapOutputKeyClass());
jobConf.setMapOutputValueClass(IndexUpdateMapper.getMapOutputValueClass());
jobConf.setOutputKeyClass(IndexUpdateReducer.getOutputKeyClass());
jobConf.setOutputValueClass(IndexUpdateReducer.getOutputValueClass());
jobConf.setMapperClass(IndexUpdateMapper.class);
jobConf.setPartitionerClass(IndexUpdatePartitioner.class);
jobConf.setCombinerClass(IndexUpdateCombiner.class);
jobConf.setReducerClass(IndexUpdateReducer.class);
jobConf.setOutputFormat(IndexUpdateOutputFormat.class);
return jobConf;
If distributed cache is modified in dfs
between two job runs, it can be present more than once
in any of the task tracker, in which job ran.
SATD_ADDED
testDistributedCache()
public void testDistributedCache() throws Exception
Configuration conf = new Configuration(cluster.getConf());
JTProtocol wovenClient = cluster.getJTClient().getProxy();
// This counter will check for count of a loop,
// which might become infinite.
int count = 0;
// This boolean will decide whether to run job again
boolean continueLoop = true;
// counter for job Loop
int countLoop = 0;
// This counter increases with all the tasktrackers in which tasks ran
int taskTrackerCounter = 0;
// This will store all the tasktrackers in which tasks ran
ArrayList taskTrackerCollection = new ArrayList();
// This boolean tells if two tasks ran onteh same tasktracker or not
boolean taskTrackerFound = false;
do {
SleepJob job = new SleepJob();
job.setConf(conf);
Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100);
// Before starting, Modify the file
String input = "This will be the content of\n" + "distributed cache\n";
// Creating the path with the file
DataOutputStream file = UtilsForTests.createTmpFileDFS(dfs, URIPATH, permission, input);
DistributedCache.createSymlink(conf);
URI uri = URI.create(uriPath);
DistributedCache.addCacheFile(uri, conf);
JobConf jconf = new JobConf(conf);
// Controls the job till all verification is done
FinishTaskControlAction.configureControlActionForJob(conf);
slpJob.submit();
// Submitting the job
RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));
// counter for job Loop
countLoop++;
TTClient tClient = null;
JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
LOG.info("jInfo is :" + jInfo);
// Assert if jobInfo is null
Assert.assertNotNull("jobInfo is null", jInfo);
// Wait for the job to start running.
count = 0;
while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
UtilsForTests.waitFor(10000);
count++;
jInfo = wovenClient.getJobInfo(rJob.getID());
// If the count goes beyond a point, then break; This is to avoid
// infinite loop under unforeseen circumstances. Testcase will anyway
// fail later.
if (count > 10) {
Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point");
}
}
LOG.info("job id is :" + rJob.getID().toString());
TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID());
boolean distCacheFileIsFound;
for (TaskInfo taskInfo : taskInfos) {
distCacheFileIsFound = false;
String[] taskTrackers = taskInfo.getTaskTrackers();
for (String taskTracker : taskTrackers) {
// Formatting tasktracker to get just its FQDN
taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
LOG.info("taskTracker is :" + taskTracker);
// The tasktrackerFound variable is initialized
taskTrackerFound = false;
// This will be entered from the second job onwards
if (countLoop > 1) {
if (taskTracker != null) {
continueLoop = taskTrackerCollection.contains(taskTracker);
}
if (continueLoop) {
taskTrackerFound = true;
}
}
// Collecting the tasktrackers
if (taskTracker != null)
taskTrackerCollection.add(taskTracker);
// we have loopped through two times to look for task
// getting submitted on same tasktrackers.The same tasktracker
// for subsequent jobs was not hit maybe because of many number
// of tasktrackers. So, testcase has to stop here.
if (countLoop > 1) {
continueLoop = false;
}
tClient = cluster.getTTClient(taskTracker);
// tClient maybe null because the task is already dead. Ex: setup
if (tClient == null) {
continue;
}
String[] localDirs = tClient.getMapredLocalDirs();
int distributedFileCount = 0;
// Go to every single path
for (String localDir : localDirs) {
// Public Distributed cache will always be stored under
// mapre.local.dir/tasktracker/archive
localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir();
LOG.info("localDir is : " + localDir);
// Get file status of all the directories
// and files under that path.
FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true);
for (FileStatus fileStatus : fileStatuses) {
Path path = fileStatus.getPath();
LOG.info("path is :" + path.toString());
// Checking if the received path ends with
// the distributed filename
distCacheFileIsFound = (path.toString()).endsWith(distributedFileName);
// If file is found, check for its permission.
// Since the file is found break out of loop
if (distCacheFileIsFound) {
LOG.info("PATH found is :" + path.toString());
distributedFileCount++;
String filename = path.getName();
FsPermission fsPerm = fileStatus.getPermission();
Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777")));
}
}
}
LOG.debug("The distributed FileCount is :" + distributedFileCount);
LOG.debug("The taskTrackerFound is :" + taskTrackerFound);
// If distributed cache is modified in dfs
// between two job runs, it can be present more than once
// in any of the task tracker, in which job ran.
if (distributedFileCount != 2 && taskTrackerFound) {
Assert.fail("The distributed cache file has to be two. " + "But found was " + distributedFileCount);
} else if (distributedFileCount > 1 && !taskTrackerFound) {
Assert.fail("The distributed cache file cannot more than one." + " But found was " + distributedFileCount);
} else if (distributedFileCount < 1)
Assert.fail("The distributed cache file is less than one. " + "But found was " + distributedFileCount);
if (!distCacheFileIsFound) {
Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false);
}
}
}
// Allow the job to continue through MR control job.
for (TaskInfo taskInfoRemaining : taskInfos) {
FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID()));
Collection tts = cluster.getTTClients();
for (TTClient cli : tts) {
cli.getProxy().sendAction(action);
}
}
// Killing the job because all the verification needed
// for this testcase is completed.
rJob.killJob();
// Waiting for 3 seconds for cleanup to start
Thread.sleep(3000);
// Getting the last cleanup task's tasktracker also, as
// distributed cache gets uploaded even during cleanup.
TaskInfo[] myTaskInfos = wovenClient.getTaskInfo(rJob.getID());
if (myTaskInfos != null) {
for (TaskInfo info : myTaskInfos) {
if (info.isSetupOrCleanup()) {
String[] taskTrackers = info.getTaskTrackers();
for (String taskTracker : taskTrackers) {
// Formatting tasktracker to get just its FQDN
taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
LOG.info("taskTracker is :" + taskTracker);
// Collecting the tasktrackers
if (taskTracker != null)
taskTrackerCollection.add(taskTracker);
}
}
}
}
// Making sure that the job is complete.
while (jInfo != null && !jInfo.getStatus().isJobComplete()) {
Thread.sleep(10000);
jInfo = wovenClient.getJobInfo(rJob.getID());
}
} while (continueLoop);
tClient maybe null because the task is already dead. Ex: setup
SATD_ADDED
testDistributedCache()
public void testDistributedCache() throws Exception
Configuration conf = new Configuration(cluster.getConf());
JTProtocol wovenClient = cluster.getJTClient().getProxy();
// This counter will check for count of a loop,
// which might become infinite.
int count = 0;
// This boolean will decide whether to run job again
boolean continueLoop = true;
// counter for job Loop
int countLoop = 0;
// This counter increases with all the tasktrackers in which tasks ran
int taskTrackerCounter = 0;
// This will store all the tasktrackers in which tasks ran
ArrayList taskTrackerCollection = new ArrayList();
// This boolean tells if two tasks ran onteh same tasktracker or not
boolean taskTrackerFound = false;
do {
SleepJob job = new SleepJob();
job.setConf(conf);
Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100);
// Before starting, Modify the file
String input = "This will be the content of\n" + "distributed cache\n";
// Creating the path with the file
DataOutputStream file = UtilsForTests.createTmpFileDFS(dfs, URIPATH, permission, input);
DistributedCache.createSymlink(conf);
URI uri = URI.create(uriPath);
DistributedCache.addCacheFile(uri, conf);
JobConf jconf = new JobConf(conf);
// Controls the job till all verification is done
FinishTaskControlAction.configureControlActionForJob(conf);
slpJob.submit();
// Submitting the job
RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));
// counter for job Loop
countLoop++;
TTClient tClient = null;
JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
LOG.info("jInfo is :" + jInfo);
// Assert if jobInfo is null
Assert.assertNotNull("jobInfo is null", jInfo);
// Wait for the job to start running.
count = 0;
while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
UtilsForTests.waitFor(10000);
count++;
jInfo = wovenClient.getJobInfo(rJob.getID());
// If the count goes beyond a point, then break; This is to avoid
// infinite loop under unforeseen circumstances. Testcase will anyway
// fail later.
if (count > 10) {
Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point");
}
}
LOG.info("job id is :" + rJob.getID().toString());
TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID());
boolean distCacheFileIsFound;
for (TaskInfo taskInfo : taskInfos) {
distCacheFileIsFound = false;
String[] taskTrackers = taskInfo.getTaskTrackers();
for (String taskTracker : taskTrackers) {
// Formatting tasktracker to get just its FQDN
taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
LOG.info("taskTracker is :" + taskTracker);
// The tasktrackerFound variable is initialized
taskTrackerFound = false;
// This will be entered from the second job onwards
if (countLoop > 1) {
if (taskTracker != null) {
continueLoop = taskTrackerCollection.contains(taskTracker);
}
if (continueLoop) {
taskTrackerFound = true;
}
}
// Collecting the tasktrackers
if (taskTracker != null)
taskTrackerCollection.add(taskTracker);
// we have loopped through two times to look for task
// getting submitted on same tasktrackers.The same tasktracker
// for subsequent jobs was not hit maybe because of many number
// of tasktrackers. So, testcase has to stop here.
if (countLoop > 1) {
continueLoop = false;
}
tClient = cluster.getTTClient(taskTracker);
// tClient maybe null because the task is already dead. Ex: setup
if (tClient == null) {
continue;
}
String[] localDirs = tClient.getMapredLocalDirs();
int distributedFileCount = 0;
// Go to every single path
for (String localDir : localDirs) {
// Public Distributed cache will always be stored under
// mapre.local.dir/tasktracker/archive
localDir = localDir + Path.SEPARATOR + TaskTracker.getPublicDistributedCacheDir();
LOG.info("localDir is : " + localDir);
// Get file status of all the directories
// and files under that path.
FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true);
for (FileStatus fileStatus : fileStatuses) {
Path path = fileStatus.getPath();
LOG.info("path is :" + path.toString());
// Checking if the received path ends with
// the distributed filename
distCacheFileIsFound = (path.toString()).endsWith(distributedFileName);
// If file is found, check for its permission.
// Since the file is found break out of loop
if (distCacheFileIsFound) {
LOG.info("PATH found is :" + path.toString());
distributedFileCount++;
String filename = path.getName();
FsPermission fsPerm = fileStatus.getPermission();
Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777")));
}
}
}
LOG.debug("The distributed FileCount is :" + distributedFileCount);
LOG.debug("The taskTrackerFound is :" + taskTrackerFound);
// If distributed cache is modified in dfs
// between two job runs, it can be present more than once
// in any of the task tracker, in which job ran.
if (distributedFileCount != 2 && taskTrackerFound) {
Assert.fail("The distributed cache file has to be two. " + "But found was " + distributedFileCount);
} else if (distributedFileCount > 1 && !taskTrackerFound) {
Assert.fail("The distributed cache file cannot more than one." + " But found was " + distributedFileCount);
} else if (distributedFileCount < 1)
Assert.fail("The distributed cache file is less than one. " + "But found was " + distributedFileCount);
if (!distCacheFileIsFound) {
Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false);
}
}
}
// Allow the job to continue through MR control job.
for (TaskInfo taskInfoRemaining : taskInfos) {
FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID()));
Collection tts = cluster.getTTClients();
for (TTClient cli : tts) {
cli.getProxy().sendAction(action);
}
}
// Killing the job because all the verification needed
// for this testcase is completed.
rJob.killJob();
// Waiting for 3 seconds for cleanup to start
Thread.sleep(3000);
// Getting the last cleanup task's tasktracker also, as
// distributed cache gets uploaded even during cleanup.
TaskInfo[] myTaskInfos = wovenClient.getTaskInfo(rJob.getID());
if (myTaskInfos != null) {
for (TaskInfo info : myTaskInfos) {
if (info.isSetupOrCleanup()) {
String[] taskTrackers = info.getTaskTrackers();
for (String taskTracker : taskTrackers) {
// Formatting tasktracker to get just its FQDN
taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
LOG.info("taskTracker is :" + taskTracker);
// Collecting the tasktrackers
if (taskTracker != null)
taskTrackerCollection.add(taskTracker);
}
}
}
}
// Making sure that the job is complete.
while (jInfo != null && !jInfo.getStatus().isJobComplete()) {
Thread.sleep(10000);
jInfo = wovenClient.getJobInfo(rJob.getID());
}
} while (continueLoop);
When step == 0, this loop starts as many map tasks it can wrt
maxTasksPerJob
When step == 1, this loop starts as many reduce tasks it can wrt
maxTasksPerJob
When step == 2, this loop starts as many map tasks it can
When step == 3, this loop starts as many reduce tasks it can
It may seem that we would improve this loop by queuing jobs we cannot
start in steps 0 and 1 because of maxTasksPerJob, and using that queue
in step 2 and 3.
A first thing to notice is that the time with the current algorithm is
logarithmic, because it is the sum of (p^k) for k from 1 to N, were
N is the number of jobs and p is the probability for a job to not exceed
limits The probability for the cache to be useful would be similar to
p^N, that is 1/(e^N), whereas its size and the time spent to manage it
would be in ln(N).
So it is not a good idea.
SATD_ADDED
assignTasks(TaskTracker)
public synchronized List assignTasks(TaskTracker taskTracker) throws IOException
TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
final int numTaskTrackers = taskTrackerManager.getClusterStatus().getTaskTrackers();
Collection jobQueue = jobQueueJobInProgressListener.getJobQueue();
Task task;
/* Stats about the current taskTracker */
final int mapTasksNumber = taskTrackerStatus.countMapTasks();
final int reduceTasksNumber = taskTrackerStatus.countReduceTasks();
final int maximumMapTasksNumber = taskTrackerStatus.getMaxMapSlots();
final int maximumReduceTasksNumber = taskTrackerStatus.getMaxReduceSlots();
/*
* Statistics about the whole cluster. Most are approximate because of
* concurrency
*/
final int[] maxMapAndReduceLoad = getMaxMapAndReduceLoad(maximumMapTasksNumber, maximumReduceTasksNumber);
final int maximumMapLoad = maxMapAndReduceLoad[0];
final int maximumReduceLoad = maxMapAndReduceLoad[1];
final int beginAtStep;
/*
* When step == 0, this loop starts as many map tasks it can wrt
* maxTasksPerJob
* When step == 1, this loop starts as many reduce tasks it can wrt
* maxTasksPerJob
* When step == 2, this loop starts as many map tasks it can
* When step == 3, this loop starts as many reduce tasks it can
*
* It may seem that we would improve this loop by queuing jobs we cannot
* start in steps 0 and 1 because of maxTasksPerJob, and using that queue
* in step 2 and 3.
* A first thing to notice is that the time with the current algorithm is
* logarithmic, because it is the sum of (p^k) for k from 1 to N, were
* N is the number of jobs and p is the probability for a job to not exceed
* limits The probability for the cache to be useful would be similar to
* p^N, that is 1/(e^N), whereas its size and the time spent to manage it
* would be in ln(N).
* So it is not a good idea.
*/
if (maxTasksPerJob != Long.MAX_VALUE) {
beginAtStep = 0;
} else {
beginAtStep = 2;
}
List assignedTasks = new ArrayList();
scheduleTasks: for (int step = beginAtStep; step <= 3; ++step) {
/* If we reached the maximum load for this step, go to the next */
if ((step == 0 || step == 2) && mapTasksNumber >= maximumMapLoad || (step == 1 || step == 3) && reduceTasksNumber >= maximumReduceLoad) {
continue;
}
/* For each job, start its tasks */
synchronized (jobQueue) {
for (JobInProgress job : jobQueue) {
/* Ignore non running jobs */
if (job.getStatus().getRunState() != JobStatus.RUNNING) {
continue;
}
/* Check that we're not exceeding the global limits */
if ((step == 0 || step == 1) && (job.runningMaps() + job.runningReduces() >= maxTasksPerJob)) {
continue;
}
if (step == 0 || step == 2) {
task = job.obtainNewMapTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
} else {
task = job.obtainNewReduceTask(taskTrackerStatus, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
}
if (task != null) {
assignedTasks.add(task);
break scheduleTasks;
}
}
}
}
return assignedTasks;
public static boolean injectCriteria(String klassName)
boolean trigger = false;
// TODO fix this: make it more sophisticated!!!
if (generator.nextFloat() < getProbability(klassName)) {
trigger = true;
}
return trigger;
Set new default probability if specified through a system.property
If neither is specified set default probability to DEFAULT_PROB
SATD_ADDED
injectCriteria(String)
public static boolean injectCriteria(String klassName)
boolean trigger = false;
// TODO fix this: make it more sophisticated!!!
if (generator.nextFloat() < getProbability(klassName)) {
trigger = true;
}
return trigger;
TODO: Support BINARY, VARBINARY, LONGVARBINARY, DISTINCT, CLOB, BLOB, ARRAY
STRUCT, REF, DATALINK, and JAVA_OBJECT.
SATD_ADDED
getSplitter(int)
protected DBSplitter getSplitter(int sqlDataType)
switch(sqlDataType) {
case Types.NUMERIC:
case Types.DECIMAL:
return new BigDecimalSplitter();
case Types.BIT:
case Types.BOOLEAN:
return new BooleanSplitter();
case Types.INTEGER:
case Types.TINYINT:
case Types.SMALLINT:
case Types.BIGINT:
return new IntegerSplitter();
case Types.REAL:
case Types.FLOAT:
case Types.DOUBLE:
return new FloatSplitter();
case Types.CHAR:
case Types.VARCHAR:
case Types.LONGVARCHAR:
return new TextSplitter();
case Types.DATE:
case Types.TIME:
case Types.TIMESTAMP:
return new DateSplitter();
default:
// TODO: Support BINARY, VARBINARY, LONGVARBINARY, DISTINCT, CLOB, BLOB, ARRAY
// STRUCT, REF, DATALINK, and JAVA_OBJECT.
return null;
}
public void testSuccessiveVolumeFailures() throws Exception
assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
// Bring up two more datanodes
cluster.startDataNodes(conf, 2, true, null, null);
cluster.waitActive();
/*
* Calculate the total capacity of all the datanodes. Sleep for
* three seconds to be sure the datanodes have had a chance to
* heartbeat their capacities.
*/
Thread.sleep(WAIT_FOR_HEARTBEATS);
FSNamesystem ns = cluster.getNamesystem();
long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns);
long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0);
File dn1Vol1 = new File(dataDir, "data" + (2 * 0 + 1));
File dn2Vol1 = new File(dataDir, "data" + (2 * 1 + 1));
File dn3Vol1 = new File(dataDir, "data" + (2 * 2 + 1));
File dn3Vol2 = new File(dataDir, "data" + (2 * 2 + 2));
/*
* Make the 1st volume directories on the first two datanodes
* non-accessible. We don't make all three 1st volume directories
* readonly since that would cause the entire pipeline to
* fail. The client does not retry failed nodes even though
* perhaps they could succeed because just a single volume failed.
*/
assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false));
assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false));
/*
* Create file1 and wait for 3 replicas (ie all DNs can still
* store a block). Then assert that all DNs are up, despite the
* volume failures.
*/
Path file1 = new Path("/test1");
DFSTestUtil.createFile(fs, file1, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file1, (short) 3);
ArrayList dns = cluster.getDataNodes();
assertTrue("DN1 should be up", dns.get(0).isDatanodeUp());
assertTrue("DN2 should be up", dns.get(1).isDatanodeUp());
assertTrue("DN3 should be up", dns.get(2).isDatanodeUp());
/*
* The metrics should confirm the volume failures.
*/
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(0).getMetrics().name()));
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(1).getMetrics().name()));
assertCounter("VolumeFailures", 0L, getMetrics(dns.get(2).getMetrics().name()));
// Ensure we wait a sufficient amount of time
assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
// Eventually the NN should report two volume failures
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* Now fail a volume on the third datanode. We should be able to get
* three replicas since we've already identified the other failures.
*/
assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(false));
Path file2 = new Path("/test2");
DFSTestUtil.createFile(fs, file2, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file2, (short) 3);
assertTrue("DN3 should still be up", dns.get(2).isDatanodeUp());
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(2).getMetrics().name()));
ArrayList live = new ArrayList();
ArrayList dead = new ArrayList();
ns.DFSNodesStatus(live, dead);
live.clear();
dead.clear();
ns.DFSNodesStatus(live, dead);
assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures());
/*
* Once the datanodes have a chance to heartbeat their new capacity the
* total capacity should be down by three volumes (assuming the host
* did not grow or shrink the data volume while the test was running).
*/
dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0);
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, origCapacity - (3 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* Now fail the 2nd volume on the 3rd datanode. All its volumes
* are now failed and so it should report two volume failures
* and that it's no longer up. Only wait for two replicas since
* we'll never get a third.
*/
assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(false));
Path file3 = new Path("/test3");
DFSTestUtil.createFile(fs, file3, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file3, (short) 2);
// The DN should consider itself dead
DFSTestUtil.waitForDatanodeDeath(dns.get(2));
// And report two failed volumes
assertCounter("VolumeFailures", 2L, getMetrics(dns.get(2).getMetrics().name()));
// The NN considers the DN dead
DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, origCapacity - (4 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* The datanode never tries to restore the failed volume, even if
* it's subsequently repaired, but it should see this volume on
* restart, so file creation should be able to succeed after
* restoring the data directories and restarting the datanodes.
*/
assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(true));
cluster.restartDataNodes();
cluster.waitActive();
Path file4 = new Path("/test4");
DFSTestUtil.createFile(fs, file4, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file4, (short) 3);
/*
* Eventually the capacity should be restored to its original value,
* and that the volume failure count should be reported as zero by
* both the metrics and the NN.
*/
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS);
Eventually the NN should report two volume failures
SATD_ADDED
testSuccessiveVolumeFailures()
public void testSuccessiveVolumeFailures() throws Exception
assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
// Bring up two more datanodes
cluster.startDataNodes(conf, 2, true, null, null);
cluster.waitActive();
/*
* Calculate the total capacity of all the datanodes. Sleep for
* three seconds to be sure the datanodes have had a chance to
* heartbeat their capacities.
*/
Thread.sleep(WAIT_FOR_HEARTBEATS);
FSNamesystem ns = cluster.getNamesystem();
long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns);
long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0);
File dn1Vol1 = new File(dataDir, "data" + (2 * 0 + 1));
File dn2Vol1 = new File(dataDir, "data" + (2 * 1 + 1));
File dn3Vol1 = new File(dataDir, "data" + (2 * 2 + 1));
File dn3Vol2 = new File(dataDir, "data" + (2 * 2 + 2));
/*
* Make the 1st volume directories on the first two datanodes
* non-accessible. We don't make all three 1st volume directories
* readonly since that would cause the entire pipeline to
* fail. The client does not retry failed nodes even though
* perhaps they could succeed because just a single volume failed.
*/
assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false));
assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false));
/*
* Create file1 and wait for 3 replicas (ie all DNs can still
* store a block). Then assert that all DNs are up, despite the
* volume failures.
*/
Path file1 = new Path("/test1");
DFSTestUtil.createFile(fs, file1, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file1, (short) 3);
ArrayList dns = cluster.getDataNodes();
assertTrue("DN1 should be up", dns.get(0).isDatanodeUp());
assertTrue("DN2 should be up", dns.get(1).isDatanodeUp());
assertTrue("DN3 should be up", dns.get(2).isDatanodeUp());
/*
* The metrics should confirm the volume failures.
*/
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(0).getMetrics().name()));
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(1).getMetrics().name()));
assertCounter("VolumeFailures", 0L, getMetrics(dns.get(2).getMetrics().name()));
// Ensure we wait a sufficient amount of time
assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
// Eventually the NN should report two volume failures
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* Now fail a volume on the third datanode. We should be able to get
* three replicas since we've already identified the other failures.
*/
assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(false));
Path file2 = new Path("/test2");
DFSTestUtil.createFile(fs, file2, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file2, (short) 3);
assertTrue("DN3 should still be up", dns.get(2).isDatanodeUp());
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(2).getMetrics().name()));
ArrayList live = new ArrayList();
ArrayList dead = new ArrayList();
ns.DFSNodesStatus(live, dead);
live.clear();
dead.clear();
ns.DFSNodesStatus(live, dead);
assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures());
/*
* Once the datanodes have a chance to heartbeat their new capacity the
* total capacity should be down by three volumes (assuming the host
* did not grow or shrink the data volume while the test was running).
*/
dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0);
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, origCapacity - (3 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* Now fail the 2nd volume on the 3rd datanode. All its volumes
* are now failed and so it should report two volume failures
* and that it's no longer up. Only wait for two replicas since
* we'll never get a third.
*/
assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(false));
Path file3 = new Path("/test3");
DFSTestUtil.createFile(fs, file3, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file3, (short) 2);
// The DN should consider itself dead
DFSTestUtil.waitForDatanodeDeath(dns.get(2));
// And report two failed volumes
assertCounter("VolumeFailures", 2L, getMetrics(dns.get(2).getMetrics().name()));
// The NN considers the DN dead
DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, origCapacity - (4 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* The datanode never tries to restore the failed volume, even if
* it's subsequently repaired, but it should see this volume on
* restart, so file creation should be able to succeed after
* restoring the data directories and restarting the datanodes.
*/
assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(true));
cluster.restartDataNodes();
cluster.waitActive();
Path file4 = new Path("/test4");
DFSTestUtil.createFile(fs, file4, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file4, (short) 3);
/*
* Eventually the capacity should be restored to its original value,
* and that the volume failure count should be reported as zero by
* both the metrics and the NN.
*/
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS);
Make the 1st volume directories on the first two datanodes
non-accessible. We don't make all three 1st volume directories
readonly since that would cause the entire pipeline to
fail. The client does not retry failed nodes even though
perhaps they could succeed because just a single volume failed.
SATD_ADDED
testSuccessiveVolumeFailures()
public void testSuccessiveVolumeFailures() throws Exception
assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
// Bring up two more datanodes
cluster.startDataNodes(conf, 2, true, null, null);
cluster.waitActive();
/*
* Calculate the total capacity of all the datanodes. Sleep for
* three seconds to be sure the datanodes have had a chance to
* heartbeat their capacities.
*/
Thread.sleep(WAIT_FOR_HEARTBEATS);
FSNamesystem ns = cluster.getNamesystem();
long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(ns);
long dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0);
File dn1Vol1 = new File(dataDir, "data" + (2 * 0 + 1));
File dn2Vol1 = new File(dataDir, "data" + (2 * 1 + 1));
File dn3Vol1 = new File(dataDir, "data" + (2 * 2 + 1));
File dn3Vol2 = new File(dataDir, "data" + (2 * 2 + 2));
/*
* Make the 1st volume directories on the first two datanodes
* non-accessible. We don't make all three 1st volume directories
* readonly since that would cause the entire pipeline to
* fail. The client does not retry failed nodes even though
* perhaps they could succeed because just a single volume failed.
*/
assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(false));
assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(false));
/*
* Create file1 and wait for 3 replicas (ie all DNs can still
* store a block). Then assert that all DNs are up, despite the
* volume failures.
*/
Path file1 = new Path("/test1");
DFSTestUtil.createFile(fs, file1, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file1, (short) 3);
ArrayList dns = cluster.getDataNodes();
assertTrue("DN1 should be up", dns.get(0).isDatanodeUp());
assertTrue("DN2 should be up", dns.get(1).isDatanodeUp());
assertTrue("DN3 should be up", dns.get(2).isDatanodeUp());
/*
* The metrics should confirm the volume failures.
*/
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(0).getMetrics().name()));
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(1).getMetrics().name()));
assertCounter("VolumeFailures", 0L, getMetrics(dns.get(2).getMetrics().name()));
// Ensure we wait a sufficient amount of time
assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
// Eventually the NN should report two volume failures
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 2, origCapacity - (1 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* Now fail a volume on the third datanode. We should be able to get
* three replicas since we've already identified the other failures.
*/
assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(false));
Path file2 = new Path("/test2");
DFSTestUtil.createFile(fs, file2, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file2, (short) 3);
assertTrue("DN3 should still be up", dns.get(2).isDatanodeUp());
assertCounter("VolumeFailures", 1L, getMetrics(dns.get(2).getMetrics().name()));
ArrayList live = new ArrayList();
ArrayList dead = new ArrayList();
ns.DFSNodesStatus(live, dead);
live.clear();
dead.clear();
ns.DFSNodesStatus(live, dead);
assertEquals("DN3 should have 1 failed volume", 1, live.get(2).getVolumeFailures());
/*
* Once the datanodes have a chance to heartbeat their new capacity the
* total capacity should be down by three volumes (assuming the host
* did not grow or shrink the data volume while the test was running).
*/
dnCapacity = DFSTestUtil.getDatanodeCapacity(ns, 0);
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 3, origCapacity - (3 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* Now fail the 2nd volume on the 3rd datanode. All its volumes
* are now failed and so it should report two volume failures
* and that it's no longer up. Only wait for two replicas since
* we'll never get a third.
*/
assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(false));
Path file3 = new Path("/test3");
DFSTestUtil.createFile(fs, file3, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file3, (short) 2);
// The DN should consider itself dead
DFSTestUtil.waitForDatanodeDeath(dns.get(2));
// And report two failed volumes
assertCounter("VolumeFailures", 2L, getMetrics(dns.get(2).getMetrics().name()));
// The NN considers the DN dead
DFSTestUtil.waitForDatanodeStatus(ns, 2, 1, 2, origCapacity - (4 * dnCapacity), WAIT_FOR_HEARTBEATS);
/*
* The datanode never tries to restore the failed volume, even if
* it's subsequently repaired, but it should see this volume on
* restart, so file creation should be able to succeed after
* restoring the data directories and restarting the datanodes.
*/
assertTrue("Couldn't chmod local vol", dn1Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn2Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn3Vol1.setExecutable(true));
assertTrue("Couldn't chmod local vol", dn3Vol2.setExecutable(true));
cluster.restartDataNodes();
cluster.waitActive();
Path file4 = new Path("/test4");
DFSTestUtil.createFile(fs, file4, 1024, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file4, (short) 3);
/*
* Eventually the capacity should be restored to its original value,
* and that the volume failure count should be reported as zero by
* both the metrics and the NN.
*/
DFSTestUtil.waitForDatanodeStatus(ns, 3, 0, 0, origCapacity, WAIT_FOR_HEARTBEATS);
If TT has Map and Reduce slot free, we assign 1 map and 1 reduce
We base decision on how much is needed
versus how much is used
SATD_ADDED
assignTasks(TaskTracker)
public synchronized List assignTasks(TaskTracker taskTracker) throws IOException
TaskLookupResult tlr;
TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
List result = new ArrayList();
/*
* If TT has Map and Reduce slot free, we assign 1 map and 1 reduce
* We base decision on how much is needed
* versus how much is used
*/
ClusterStatus c = taskTrackerManager.getClusterStatus();
int mapClusterCapacity = c.getMaxMapTasks();
int reduceClusterCapacity = c.getMaxReduceTasks();
int maxMapSlots = taskTrackerStatus.getMaxMapSlots();
int currentMapSlots = taskTrackerStatus.countOccupiedMapSlots();
int maxReduceSlots = taskTrackerStatus.getMaxReduceSlots();
int currentReduceSlots = taskTrackerStatus.countOccupiedReduceSlots();
if (LOG.isDebugEnabled()) {
LOG.debug("TT asking for task, max maps=" + taskTrackerStatus.getMaxMapSlots() + ", run maps=" + taskTrackerStatus.countMapTasks() + ", max reds=" + taskTrackerStatus.getMaxReduceSlots() + ", run reds=" + taskTrackerStatus.countReduceTasks() + ", map cap=" + mapClusterCapacity + ", red cap = " + reduceClusterCapacity);
}
/*
* update all our QSC objects.
* This involves updating each qsC structure. This operation depends
* on the number of running jobs in a queue, and some waiting jobs. If it
* becomes expensive, do it once every few heartbeats only.
*/
updateContextObjects(mapClusterCapacity, reduceClusterCapacity);
// make sure we get our map or reduce scheduling object to update its
// collection of QSC objects too.
if (maxReduceSlots > currentReduceSlots) {
// reduce slot available , try to get a
// reduce task
tlr = reduceScheduler.assignTasks(taskTracker);
if (TaskLookupResult.LookUpStatus.TASK_FOUND == tlr.getLookUpStatus()) {
result.add(tlr.getTask());
}
}
if (maxMapSlots > currentMapSlots) {
// map slot available , try to get a map task
tlr = mapScheduler.assignTasks(taskTracker);
if (TaskLookupResult.LookUpStatus.TASK_FOUND == tlr.getLookUpStatus()) {
result.add(tlr.getTask());
}
}
return (result.isEmpty()) ? null : result;
we do not 'reserve' slots on tasktrackers anymore since the user is
already over the limit
Note: some of the code from above is repeated here. This is on
purpose as it improves overall readability.
Note: we walk through jobs again. Some of these jobs, which weren't
considered in the first pass, shouldn't be considered here again,
but we still check for their viability to keep the code simple. In
some cases, for high mem jobs that have nothing to run, we call
obtainNewTask() unnecessarily. Should this be a problem, we can
create a list of jobs to look at (those whose users were over
limit) in the first pass and walk through that list only.
SATD_ADDED
toString()
public String toString()
// note that we do not call updateContextObjects() here for performance
// reasons. This means that the data we print out may be slightly
// stale. This data is updated whenever assignTasks() is called
// If this doesn't happen, the data gets stale. If we see
// this often, we may need to detect this situation and call
// updateContextObjects(), or just call it each time.
return scheduler.getDisplayInfo(queueName);
look at how much capacity they've filled. Treat a queue with
capacity=0 equivalent to a queue running at capacity
SATD_ADDED
toString()
public String toString()
// note that we do not call updateContextObjects() here for performance
// reasons. This means that the data we print out may be slightly
// stale. This data is updated whenever assignTasks() is called
// If this doesn't happen, the data gets stale. If we see
// this often, we may need to detect this situation and call
// updateContextObjects(), or just call it each time.
return scheduler.getDisplayInfo(queueName);
TODO: move it to DatanodeID once HADOOP-2797 has been committed
SATD_ADDED
readFields(DataInput)
public void readFields(DataInput in) throws IOException
super.readFields(in);
// TODO: move it to DatanodeID once HADOOP-2797 has been committed
this.ipcPort = in.readShort() & 0x0000ffff;
storageInfo.readFields(in);
exportedKeys.readFields(in);
TODO: move it to DatanodeID once HADOOP-2797 has been committed
SATD_ADDED
write(DataOutput)
public void write(DataOutput out) throws IOException
super.write(out);
// TODO: move it to DatanodeID once HADOOP-2797 has been committed
out.writeShort(ipcPort);
storageInfo.write(out);
exportedKeys.write(out);
LOG.info("Testing job-success");
Path inDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/input");
Path outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output");
job.setNumMapTasks(1);
job.setNumReduceTasks(0);
// submit the job
RunningJob rJob = UtilsForTests.runJobSucceed(job, inDir, outDir);
// wait for the job to be successful
rJob.waitForCompletion();
// check if the job success was notified
assertFalse("Missing event notification for a successful job", myListener.contains(rJob.getID()));
// check if successful
assertEquals("Job failed!", JobStatus.SUCCEEDED, rJob.getJobState());
// test if 0-task jobs with setup-cleanup works fine
LOG.info("Testing job with no task job with setup and cleanup");
job.setNumMapTasks(0);
job.setNumReduceTasks(0);
outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output-no-tasks");
// submit the job
rJob = UtilsForTests.runJobSucceed(job, inDir, outDir);
// wait for the job to be successful
rJob.waitForCompletion();
// check if the job success was notified
assertFalse("Missing event notification for a successful job with no tasks", myListener.contains(rJob.getID(), true));
// check if successful
assertEquals("Job failed!", JobStatus.SUCCEEDED, rJob.getJobState());
// test if jobs with no tasks (0 maps, 0 red) update the listener properly
LOG.info("Testing job with no-set-cleanup no task");
outDir = new Path(TEST_ROOT_DIR + "/jiplistenerjob/output-no-tasks-no-set");
Job j = MapReduceTestUtil.createJob(mr.createJobConf(), inDir, outDir, 0, 0);
j.setJobSetupCleanupNeeded(false);
j.setOutputFormatClass(TestNoJobSetupCleanup.MyOutputFormat.class);
j.submit();
j.waitForCompletion(true);
JobID id = JobID.downgrade(j.getJobID());
// check if the job is in the waiting queue
assertFalse("Missing event notification on no-set-cleanup no task job", myListener.contains(id, true));
// check if the job is successful
assertEquals("Job status doesnt reflect success", JobStatus.SUCCEEDED, rJob.getJobState());
add the correction factor of 234 as the input split is also streamed
SATD_ADDED
runWordCount(MiniMRCluster, JobConf)
public static void runWordCount(MiniMRCluster mr, JobConf jobConf) throws IOException
LOG.info("runWordCount");
// Run a word count example
// Keeping tasks that match this pattern
String pattern = TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
jobConf.setKeepTaskFilesPattern(pattern);
TestResult result;
final Path inDir = new Path("./wc/input");
final Path outDir = new Path("./wc/output");
String input = "The quick brown fox\nhas many silly\nred fox sox\n";
result = launchWordCount(jobConf, inDir, outDir, input, 3, 1);
assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output);
JobID jobid = result.job.getID();
TaskAttemptID taskid = new TaskAttemptID(new TaskID(jobid, TaskType.MAP, 1), 0);
String userName = UserGroupInformation.getLoginUser().getUserName();
checkTaskDirectories(mr, userName, new String[] { jobid.toString() }, new String[] { taskid.toString() });
// test with maps=0
jobConf = mr.createJobConf();
input = "owen is oom";
result = launchWordCount(jobConf, inDir, outDir, input, 0, 1);
assertEquals("is\t1\noom\t1\nowen\t1\n", result.output);
Counters counters = result.job.getCounters();
long hdfsRead = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP, Task.getFileSystemCounterNames("hdfs")[0]).getCounter();
long hdfsWrite = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP, Task.getFileSystemCounterNames("hdfs")[1]).getCounter();
long rawSplitBytesRead = counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getCounter();
assertEquals(result.output.length(), hdfsWrite);
// add the correction factor of 234 as the input split is also streamed
assertEquals(input.length() + rawSplitBytesRead, hdfsRead);
// Run a job with input and output going to localfs even though the
// default fs is hdfs.
{
FileSystem localfs = FileSystem.getLocal(jobConf);
String TEST_ROOT_DIR = new File(System.getProperty("test.build.data", "/tmp")).toString().replace(' ', '+');
Path localIn = localfs.makeQualified(new Path(TEST_ROOT_DIR + "/local/in"));
Path localOut = localfs.makeQualified(new Path(TEST_ROOT_DIR + "/local/out"));
result = launchWordCount(jobConf, localIn, localOut, "all your base belong to us", 1, 1);
assertEquals("all\t1\nbase\t1\nbelong\t1\nto\t1\nus\t1\nyour\t1\n", result.output);
assertTrue("outputs on localfs", localfs.exists(localOut));
}
// Localize job and localize task.
TaskTracker.RunningJob rjob = tracker.localizeJob(tip);
localizedJobConf = rjob.getJobConf();
if (jvmReuse) {
localizedJobConf.setNumTasksToExecutePerJvm(2);
}
initializeTask();
// TODO: Let the task run and create files.
// create files and set permissions 555. Verify if task controller sets
// the permissions for TT to delete the task dir or work dir properly
validateRemoveTaskFiles(needCleanup, jvmReuse, tip);
// Localize job and localize task.
TaskTracker.RunningJob rjob = tracker.localizeJob(tip);
localizedJobConf = rjob.getJobConf();
if (jvmReuse) {
localizedJobConf.setNumTasksToExecutePerJvm(2);
}
initializeTask();
// TODO: Let the task run and create files.
// create files and set permissions 555. Verify if task controller sets
// the permissions for TT to delete the task dir or work dir properly
validateRemoveTaskFiles(needCleanup, jvmReuse, tip);
// create files and set permissions 555. Verify if task controller sets
// the permissions for TT to delete the taskDir or workDir
String dir = (!needCleanup || jvmReuse) ? TaskTracker.getTaskWorkDir(task.getUser(), task.getJobID().toString(), taskId.toString(), task.isTaskCleanupTask()) : TaskTracker.getLocalTaskDir(task.getUser(), task.getJobID().toString(), taskId.toString(), task.isTaskCleanupTask());
Path[] paths = tracker.getLocalFiles(localizedJobConf, dir);
assertTrue("No paths found", paths.length > 0);
for (Path p : paths) {
if (tracker.getLocalFileSystem().exists(p)) {
createFileAndSetPermissions(localizedJobConf, p);
}
}
InlineCleanupQueue cleanupQueue = new InlineCleanupQueue();
tracker.setCleanupThread(cleanupQueue);
tip.removeTaskFiles(needCleanup, taskId);
if (jvmReuse) {
// work dir should still exist and cleanup queue should be empty
assertTrue("cleanup queue is not empty after removeTaskFiles() in case " + "of jvm reuse.", cleanupQueue.isQueueEmpty());
boolean workDirExists = false;
for (Path p : paths) {
if (tracker.getLocalFileSystem().exists(p)) {
workDirExists = true;
}
}
assertTrue("work dir does not exist in case of jvm reuse", workDirExists);
// now try to delete the work dir and verify that there are no stale paths
JvmManager.deleteWorkDir(tracker, task);
}
assertTrue("Some task files are not deleted!! Number of stale paths is " + cleanupQueue.stalePaths.size(), cleanupQueue.stalePaths.size() == 0);
Should have incremented the batch count exactly once
SATD_ADDED
testSyncBatching()
public void testSyncBatching() throws Exception
// start a cluster
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
ExecutorService threadA = Executors.newSingleThreadExecutor();
ExecutorService threadB = Executors.newSingleThreadExecutor();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
cluster.waitActive();
fileSys = cluster.getFileSystem();
final FSNamesystem namesystem = cluster.getNamesystem();
FSImage fsimage = namesystem.getFSImage();
final FSEditLog editLog = fsimage.getEditLog();
assertEquals("should start with no txids synced", 0, editLog.getSyncTxId());
// Log an edit from thread A
doLogEdit(threadA, editLog, "thread-a 1");
assertEquals("logging edit without syncing should do not affect txid", 0, editLog.getSyncTxId());
// Log an edit from thread B
doLogEdit(threadB, editLog, "thread-b 1");
assertEquals("logging edit without syncing should do not affect txid", 0, editLog.getSyncTxId());
// Now ask to sync edit from B, which should sync both edits.
doCallLogSync(threadB, editLog);
assertEquals("logSync from second thread should bump txid up to 2", 2, editLog.getSyncTxId());
// Now ask to sync edit from A, which was already batched in - thus
// it should increment the batch count metric
doCallLogSync(threadA, editLog);
assertEquals("logSync from first thread shouldn't change txid", 2, editLog.getSyncTxId());
// Should have incremented the batch count exactly once
assertCounter("TransactionsBatchedInSync", 1L, getMetrics("NameNodeActivity"));
} finally {
threadA.shutdown();
threadB.shutdown();
if (fileSys != null)
fileSys.close();
if (cluster != null)
cluster.shutdown();
}
Similar to BlockReport_08 but corrupts GS and len of the TEMPORARY's
replica block. Expect the same behaviour: NN should simply ignore this
block
SATD_ADDED
startUpCluster()
public void startUpCluster() throws IOException
// Reset if case a test has modified the value
REPL_FACTOR = 1;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL_FACTOR).build();
fs = (DistributedFileSystem) cluster.getFileSystem();
bpid = cluster.getNamesystem().getBlockPoolId();
now take care of the rest of the files and subdirectories
SATD_ADDED
linkBlocks(File, File, int, HardLink)
static void linkBlocks(File from, File to, int oldLV, HardLink hl) throws IOException
if (!from.exists()) {
return;
}
if (!from.isDirectory()) {
if (from.getName().startsWith(COPY_FILE_PREFIX)) {
FileInputStream in = new FileInputStream(from);
try {
FileOutputStream out = new FileOutputStream(to);
try {
IOUtils.copyBytes(in, out, 16 * 1024);
hl.linkStats.countPhysicalFileCopies++;
} finally {
out.close();
}
} finally {
in.close();
}
} else {
// check if we are upgrading from pre-generation stamp version.
if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) {
// Link to the new file name.
to = new File(convertMetatadataFileName(to.getAbsolutePath()));
}
HardLink.createHardLink(from, to);
hl.linkStats.countSingleLinks++;
}
return;
}
// from is a directory
hl.linkStats.countDirs++;
if (!to.mkdirs())
throw new IOException("Cannot create directory " + to);
// If upgrading from old stuff, need to munge the filenames. That has to
// be done one file at a time, so hardlink them one at a time (slow).
if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) {
String[] blockNames = from.list(new java.io.FilenameFilter() {
public boolean accept(File dir, String name) {
return name.startsWith(BLOCK_SUBDIR_PREFIX) || name.startsWith(BLOCK_FILE_PREFIX) || name.startsWith(COPY_FILE_PREFIX);
}
});
if (blockNames.length == 0) {
hl.linkStats.countEmptyDirs++;
} else
for (int i = 0; i < blockNames.length; i++) linkBlocks(new File(from, blockNames[i]), new File(to, blockNames[i]), oldLV, hl);
} else {
// If upgrading from a relatively new version, we only need to create
// links with the same filename. This can be done in bulk (much faster).
String[] blockNames = from.list(new java.io.FilenameFilter() {
public boolean accept(File dir, String name) {
return name.startsWith(BLOCK_FILE_PREFIX);
}
});
if (blockNames.length > 0) {
HardLink.createHardLinkMult(from, blockNames, to);
hl.linkStats.countMultLinks++;
hl.linkStats.countFilesMultLinks += blockNames.length;
} else {
hl.linkStats.countEmptyDirs++;
}
// now take care of the rest of the files and subdirectories
String[] otherNames = from.list(new java.io.FilenameFilter() {
public boolean accept(File dir, String name) {
return name.startsWith(BLOCK_SUBDIR_PREFIX) || name.startsWith(COPY_FILE_PREFIX);
}
});
for (int i = 0; i < otherNames.length; i++) linkBlocks(new File(from, otherNames[i]), new File(to, otherNames[i]), oldLV, hl);
}
// Make sure data directory exists
new File(TEST_DIR).mkdirs();
conf = new Configuration();
conf.set("raid.config.file", CONFIG_FILE);
conf.setBoolean("raid.config.reload", true);
conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
// scan all policies once every 5 second
conf.setLong("raid.policy.rescan.interval", 5000);
// make all deletions not go through Trash
conf.set("fs.shell.delete.classname", "org.apache.hadoop.dfs.DFSClient");
// the RaidNode does the raiding inline (instead of submitting to map/reduce)
if (local) {
conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
} else {
conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode");
}
conf.set("raid.server.address", "localhost:0");
// create a dfs and map-reduce cluster
final int taskTrackers = 4;
final int jobTrackerPort = 60050;
dfs = new MiniDFSCluster(conf, 3, true, null);
dfs.waitActive();
fileSys = dfs.getFileSystem();
namenode = fileSys.getUri().toString();
mr = new MiniMRCluster(taskTrackers, namenode, 3);
jobTrackerName = "localhost:" + mr.getJobTrackerPort();
hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
FileSystem.setDefaultUri(conf, namenode);
conf.set("mapred.job.tracker", jobTrackerName);
TODO: We should first fix the files that lose more blocks
SATD_ADDED
sortCorruptFiles(List)
void sortCorruptFiles(List files)
// TODO: We should first fix the files that lose more blocks
Comparator comp = new Comparator() {
public int compare(Path p1, Path p2) {
if (isXorParityFile(p2) || isRsParityFile(p2)) {
// If p2 is a parity file, p1 is smaller.
return -1;
}
if (isXorParityFile(p1) || isRsParityFile(p1)) {
// If p1 is a parity file, p2 is smaller.
return 1;
}
// If both are source files, they are equal.
return 0;
}
};
Collections.sort(files, comp);
domain name
too short
too long
too large
weird
too short
too long
too large
too large
SATD_ADDED
testIsIPAddress()
public void testIsIPAddress()
final String[] positives = { // regular ipv4
"123.13.42.255", // padded 0
"123.01.0.255", // more padded 0
"000.001.002.020", // escaped .
"123\\.13\\.42\\.255", // all-zero
"0.0.0.0", // all-0xff
"255.255.255.255", // regular ipv6
"1080:0:0:0:8:800:200C:417A", // padded 0
"1080:01:020:3:8:0800:200C:417A", // more padded 0
"1080:01:002:0003:080:0800:0200:417A", // all-zero
"0:0:0:0:0:0:0:0", // all-0xff
"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" };
final String[] negatives = { // domain name
"node.megatron.com", // too short
"13.42.255", // too long
"123.13.42.255.10", // too large
"123.256.42.255", // weird
"123.13.42.255.weird.com", // too short
"1080:0:0:0:8:200C:417A", // too long
"1080:0:0:0:1:8:800:200C:417A", // too large
"1080A:0:0:0:8:800:200C:417A", // too large
"1080:0:0:0:8:800:200G:417A" };
for (String s : positives) {
Assert.assertTrue(s, SimulatorEngine.isIPAddress(s));
}
for (String s : negatives) {
Assert.assertFalse(s, SimulatorEngine.isIPAddress(s));
}
// Make sure data directory exists
new File(TEST_DIR).mkdirs();
conf = new Configuration();
conf.set("raid.config.file", CONFIG_FILE);
conf.setBoolean("raid.config.reload", true);
conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
// scan all policies once every 5 second
conf.setLong("raid.policy.rescan.interval", 5000);
// make all deletions not go through Trash
conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
// the RaidNode does the raiding inline (instead of submitting to map/reduce)
if (local) {
conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
} else {
conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode");
}
conf.set("raid.server.address", "localhost:0");
conf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
// create a dfs and map-reduce cluster
final int taskTrackers = 4;
dfs = new MiniDFSCluster(conf, 3, true, null);
dfs.waitActive();
fileSys = dfs.getFileSystem();
namenode = fileSys.getUri().toString();
mr = new MiniMRCluster(taskTrackers, namenode, 3);
jobTrackerName = "localhost:" + mr.getJobTrackerPort();
hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
FileSystem.setDefaultUri(conf, namenode);
conf.set("mapred.job.tracker", jobTrackerName);
Make sure we really waited for the flush to complete!
SATD_ADDED
testSaveRightBeforeSync()
public void testSaveRightBeforeSync() throws Exception
Configuration conf = getConf();
NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
DFSTestUtil.formatNameNode(conf);
final FSNamesystem namesystem = new FSNamesystem(conf);
try {
FSImage fsimage = namesystem.getFSImage();
FSEditLog editLog = spy(fsimage.getEditLog());
fsimage.editLog = editLog;
final AtomicReference deferredException = new AtomicReference();
final CountDownLatch waitToEnterSync = new CountDownLatch(1);
final Thread doAnEditThread = new Thread() {
public void run() {
try {
LOG.info("Starting mkdirs");
namesystem.mkdirs("/test", new PermissionStatus("test", "test", new FsPermission((short) 00755)), true);
LOG.info("mkdirs complete");
} catch (Throwable ioe) {
deferredException.set(ioe);
waitToEnterSync.countDown();
}
}
};
Answer blockingSync = new Answer() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
LOG.info("logSync called");
if (Thread.currentThread() == doAnEditThread) {
LOG.info("edit thread: Telling main thread we made it just before logSync...");
waitToEnterSync.countDown();
LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs");
Thread.sleep(BLOCK_TIME * 1000);
LOG.info("Going through to logSync. This will allow the main thread to continue.");
}
invocation.callRealMethod();
LOG.info("logSync complete");
return null;
}
};
doAnswer(blockingSync).when(editLog).logSync();
doAnEditThread.start();
LOG.info("Main thread: waiting to just before logSync...");
waitToEnterSync.await();
assertNull(deferredException.get());
LOG.info("Main thread: detected that logSync about to be called.");
LOG.info("Trying to enter safe mode.");
LOG.info("This should block for " + BLOCK_TIME + "sec, since we have pending edits");
long st = System.currentTimeMillis();
namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
long et = System.currentTimeMillis();
LOG.info("Entered safe mode");
// Make sure we really waited for the flush to complete!
assertTrue(et - st > (BLOCK_TIME - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.saveNamespace();
LOG.info("Joining on edit thread...");
doAnEditThread.join();
assertNull(deferredException.get());
verifyEditLogs(namesystem, fsimage);
} finally {
LOG.info("Closing namesystem");
if (namesystem != null)
namesystem.close();
}
Make sure we really waited for the flush to complete!
SATD_ADDED
testSaveImageWhileSyncInProgress()
public void testSaveImageWhileSyncInProgress() throws Exception
Configuration conf = getConf();
NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
DFSTestUtil.formatNameNode(conf);
final FSNamesystem namesystem = new FSNamesystem(conf);
try {
FSImage fsimage = namesystem.getFSImage();
FSEditLog editLog = fsimage.getEditLog();
ArrayList streams = editLog.getEditStreams();
EditLogOutputStream spyElos = spy(streams.get(0));
streams.set(0, spyElos);
final AtomicReference deferredException = new AtomicReference();
final CountDownLatch waitToEnterFlush = new CountDownLatch(1);
final Thread doAnEditThread = new Thread() {
public void run() {
try {
LOG.info("Starting mkdirs");
namesystem.mkdirs("/test", new PermissionStatus("test", "test", new FsPermission((short) 00755)), true);
LOG.info("mkdirs complete");
} catch (Throwable ioe) {
deferredException.set(ioe);
waitToEnterFlush.countDown();
}
}
};
Answer blockingFlush = new Answer() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
LOG.info("Flush called");
if (Thread.currentThread() == doAnEditThread) {
LOG.info("edit thread: Telling main thread we made it to flush section...");
// Signal to main thread that the edit thread is in the racy section
waitToEnterFlush.countDown();
LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs");
Thread.sleep(BLOCK_TIME * 1000);
LOG.info("Going through to flush. This will allow the main thread to continue.");
}
invocation.callRealMethod();
LOG.info("Flush complete");
return null;
}
};
doAnswer(blockingFlush).when(spyElos).flush();
doAnEditThread.start();
// Wait for the edit thread to get to the logsync unsynchronized section
LOG.info("Main thread: waiting to enter flush...");
waitToEnterFlush.await();
assertNull(deferredException.get());
LOG.info("Main thread: detected that logSync is in unsynchronized section.");
LOG.info("Trying to enter safe mode.");
LOG.info("This should block for " + BLOCK_TIME + "sec, since flush will sleep that long");
long st = System.currentTimeMillis();
namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
long et = System.currentTimeMillis();
LOG.info("Entered safe mode");
// Make sure we really waited for the flush to complete!
assertTrue(et - st > (BLOCK_TIME - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.saveNamespace();
LOG.info("Joining on edit thread...");
doAnEditThread.join();
assertNull(deferredException.get());
verifyEditLogs(namesystem, fsimage);
} finally {
LOG.info("Closing namesystem");
if (namesystem != null)
namesystem.close();
}
synchronized (eventLog) {
eventLog.log("BEGIN_DUMP");
// List jobs in order of submit time
ArrayList jobs = new ArrayList(infos.keySet());
Collections.sort(jobs, new Comparator() {
public int compare(JobInProgress j1, JobInProgress j2) {
return (int) Math.signum(j1.getStartTime() - j2.getStartTime());
}
});
// Dump info for each job
for (JobInProgress job : jobs) {
JobProfile profile = job.getProfile();
JobInfo info = infos.get(job);
Schedulable ms = info.mapSchedulable;
Schedulable rs = info.reduceSchedulable;
eventLog.log("JOB", profile.getJobID(), profile.name, profile.user, job.getPriority(), poolMgr.getPoolName(job), job.numMapTasks, ms.getRunningTasks(), ms.getDemand(), ms.getFairShare(), ms.getWeight(), job.numReduceTasks, rs.getRunningTasks(), rs.getDemand(), rs.getFairShare(), rs.getWeight());
}
// List pools in alphabetical order
List pools = new ArrayList(poolMgr.getPools());
Collections.sort(pools, new Comparator() {
public int compare(Pool p1, Pool p2) {
if (p1.isDefaultPool())
return 1;
else if (p2.isDefaultPool())
return -1;
else
return p1.getName().compareTo(p2.getName());
}
});
for (Pool pool : pools) {
int runningMaps = 0;
int runningReduces = 0;
for (JobInProgress job : pool.getJobs()) {
JobInfo info = infos.get(job);
if (info != null) {
// TODO: Fix
// runningMaps += info.runningMaps;
// runningReduces += info.runningReduces;
}
}
String name = pool.getName();
eventLog.log("POOL", name, poolMgr.getPoolWeight(name), pool.getJobs().size(), poolMgr.getAllocation(name, TaskType.MAP), runningMaps, poolMgr.getAllocation(name, TaskType.REDUCE), runningReduces);
}
// Dump info for each pool
eventLog.log("END_DUMP");
}
Compute total map/reduce slots
In the future we can precompute this if the Scheduler becomes a
listener of tracker join/leave events.
SATD_ADDED
assignTasks(TaskTracker)
public synchronized List assignTasks(TaskTracker tracker) throws IOException
if (// Don't try to assign tasks if we haven't yet started up
!initialized)
return null;
String trackerName = tracker.getTrackerName();
eventLog.log("HEARTBEAT", trackerName);
long currentTime = clock.getTime();
// Compute total runnable maps and reduces, and currently running ones
int runnableMaps = 0;
int runningMaps = 0;
int runnableReduces = 0;
int runningReduces = 0;
for (Pool pool : poolMgr.getPools()) {
runnableMaps += pool.getMapSchedulable().getDemand();
runningMaps += pool.getMapSchedulable().getRunningTasks();
runnableReduces += pool.getReduceSchedulable().getDemand();
runningReduces += pool.getReduceSchedulable().getRunningTasks();
}
ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
// Compute total map/reduce slots
// In the future we can precompute this if the Scheduler becomes a
// listener of tracker join/leave events.
int totalMapSlots = getTotalSlots(TaskType.MAP, clusterStatus);
int totalReduceSlots = getTotalSlots(TaskType.REDUCE, clusterStatus);
eventLog.log("RUNNABLE_TASKS", runnableMaps, runningMaps, runnableReduces, runningReduces);
// Update time waited for local maps for jobs skipped on last heartbeat
updateLocalityWaitTimes(currentTime);
TaskTrackerStatus tts = tracker.getStatus();
// loop counter for map in the below while loop
int mapsAssigned = 0;
// loop counter for reduce in the below while
int reducesAssigned = 0;
int mapCapacity = maxTasksToAssign(TaskType.MAP, tts);
int reduceCapacity = maxTasksToAssign(TaskType.REDUCE, tts);
// flag used for ending the loop
boolean mapRejected = false;
// flag used for ending the loop
boolean reduceRejected = false;
// Keep track of which jobs were visited for map tasks and which had tasks
// launched, so that we can later mark skipped jobs for delay scheduling
Set visitedForMap = new HashSet();
Set visitedForReduce = new HashSet();
Set launchedMap = new HashSet();
ArrayList tasks = new ArrayList();
// Scan jobs to assign tasks until neither maps nor reduces can be assigned
while (true) {
// Computing the ending conditions for the loop
// Reject a task type if one of the following condition happens
// 1. number of assigned task reaches per heatbeat limit
// 2. number of running tasks reaches runnable tasks
// 3. task is rejected by the LoadManager.canAssign
if (!mapRejected) {
if (mapsAssigned == mapCapacity || runningMaps == runnableMaps || !loadMgr.canAssignMap(tts, runnableMaps, totalMapSlots)) {
eventLog.log("INFO", "Can't assign another MAP to " + trackerName);
mapRejected = true;
}
}
if (!reduceRejected) {
if (reducesAssigned == reduceCapacity || runningReduces == runnableReduces || !loadMgr.canAssignReduce(tts, runnableReduces, totalReduceSlots)) {
eventLog.log("INFO", "Can't assign another REDUCE to " + trackerName);
reduceRejected = true;
}
}
// Exit while (true) loop if
// 1. neither maps nor reduces can be assigned
// 2. assignMultiple is off and we already assigned one task
if (mapRejected && reduceRejected || !assignMultiple && tasks.size() > 0) {
// This is the only exit of the while (true) loop
break;
}
// Determine which task type to assign this time
// First try choosing a task type which is not rejected
TaskType taskType;
if (mapRejected) {
taskType = TaskType.REDUCE;
} else if (reduceRejected) {
taskType = TaskType.MAP;
} else {
// If both types are available, choose the task type with fewer running
// tasks on the task tracker to prevent that task type from starving
if (tts.countMapTasks() <= tts.countReduceTasks()) {
taskType = TaskType.MAP;
} else {
taskType = TaskType.REDUCE;
}
}
// Get the map or reduce schedulables and sort them by fair sharing
List scheds = getPoolSchedulables(taskType);
Collections.sort(scheds, new SchedulingAlgorithms.FairShareComparator());
boolean foundTask = false;
for (Schedulable sched : scheds) {
// This loop will assign only one task
eventLog.log("INFO", "Checking for " + taskType + " task in " + sched.getName());
Task task = taskType == TaskType.MAP ? sched.assignTask(tts, currentTime, visitedForMap) : sched.assignTask(tts, currentTime, visitedForReduce);
if (task != null) {
foundTask = true;
JobInProgress job = taskTrackerManager.getJob(task.getJobID());
eventLog.log("ASSIGN", trackerName, taskType, job.getJobID(), task.getTaskID());
// Update running task counts, and the job's locality level
if (taskType == TaskType.MAP) {
launchedMap.add(job);
mapsAssigned++;
runningMaps++;
updateLastMapLocalityLevel(job, task, tts);
} else {
reducesAssigned++;
runningReduces++;
}
// Add task to the list of assignments
tasks.add(task);
// This break makes this loop assign only one task
break;
}
// end if(task != null)
}
// end for(Schedulable sched: scheds)
// Reject the task type if we cannot find a task
if (!foundTask) {
if (taskType == TaskType.MAP) {
mapRejected = true;
} else {
reduceRejected = true;
}
}
}
// end while (true)
// Mark any jobs that were visited for map tasks but did not launch a task
// as skipped on this heartbeat
for (JobInProgress job : visitedForMap) {
if (!launchedMap.contains(job)) {
infos.get(job).skippedAtLastHeartbeat = true;
}
}
// If no tasks were found, return null
return tasks.isEmpty() ? null : tasks;
start time for virtual clock
possible improvement: set default value to sth more meaningful based on
the 1st job
SATD_ADDED
startTaskTrackers(ClusterStory, JobConf, long)
/**
* port assigned to TTs, incremented by 1 for each TT
*/
int port = 10000;
int numTaskTrackers = 0;
Random random = new Random(RandomSeedGenerator.getSeed("forStartTaskTrackers()", masterRandomSeed));
final int startDuration = jobConf.getInt("mumak.cluster.startup.duration", DEFAULT_CLUSTER_STARTUP_DURATION);
for (MachineNode node : cluster.getMachines()) {
jobConf.set("mumak.tasktracker.host.name", node.getName());
jobConf.set("mumak.tasktracker.tracker.name", "tracker_" + node.getName() + ":localhost/127.0.0.1:" + port);
long subRandomSeed = RandomSeedGenerator.getSeed("forTaskTracker" + numTaskTrackers, masterRandomSeed);
jobConf.setLong("mumak.tasktracker.random.seed", subRandomSeed);
numTaskTrackers++;
port++;
SimulatorTaskTracker tt = new SimulatorTaskTracker(jt, jobConf);
long firstHeartbeat = now + random.nextInt(startDuration);
queue.addAll(tt.init(firstHeartbeat));
}
// In startDuration + heartbeat interval of the full cluster time each
// TT is started up and told on its 2nd heartbeat to beat at a rate
// corresponding to the steady state of the cluster
long clusterSteady = now + startDuration + jt.getNextHeartbeatInterval();
return clusterSteady;
long startTaskTrackers(ClusterStory cluster, JobConf jobConf, long now)
possible improvement: change date format to human readable ?
SATD_ADDED
startTaskTrackers(ClusterStory, JobConf, long)
/**
* port assigned to TTs, incremented by 1 for each TT
*/
int port = 10000;
int numTaskTrackers = 0;
Random random = new Random(RandomSeedGenerator.getSeed("forStartTaskTrackers()", masterRandomSeed));
final int startDuration = jobConf.getInt("mumak.cluster.startup.duration", DEFAULT_CLUSTER_STARTUP_DURATION);
for (MachineNode node : cluster.getMachines()) {
jobConf.set("mumak.tasktracker.host.name", node.getName());
jobConf.set("mumak.tasktracker.tracker.name", "tracker_" + node.getName() + ":localhost/127.0.0.1:" + port);
long subRandomSeed = RandomSeedGenerator.getSeed("forTaskTracker" + numTaskTrackers, masterRandomSeed);
jobConf.setLong("mumak.tasktracker.random.seed", subRandomSeed);
numTaskTrackers++;
port++;
SimulatorTaskTracker tt = new SimulatorTaskTracker(jt, jobConf);
long firstHeartbeat = now + random.nextInt(startDuration);
queue.addAll(tt.init(firstHeartbeat));
}
// In startDuration + heartbeat interval of the full cluster time each
// TT is started up and told on its 2nd heartbeat to beat at a rate
// corresponding to the steady state of the cluster
long clusterSteady = now + startDuration + jt.getNextHeartbeatInterval();
return clusterSteady;
long startTaskTrackers(ClusterStory cluster, JobConf jobConf, long now)
List tasksToKill = new ArrayList();
List tasksToExclude = new ArrayList();
// Find tasks to kill so as to get memory usage under limits.
while (memoryStillInUsage > maxMemoryAllowedForAllTasks) {
// Exclude tasks that are already marked for killing.
// Note that we do not need to call isKillable() here because the logic
// is contained in taskTracker.findTaskToKill()
TaskInProgress task = taskTracker.findTaskToKill(tasksToExclude);
if (task == null) {
// couldn't find any more tasks to kill.
break;
}
TaskAttemptID tid = task.getTask().getTaskID();
if (processTreeInfoMap.containsKey(tid)) {
ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
memoryStillInUsage -= pTree.getCumulativeVmem();
tasksToKill.add(tid);
}
// Exclude this task from next search because it is already
// considered.
tasksToExclude.add(tid);
}
// Now kill the tasks.
if (!tasksToKill.isEmpty()) {
for (TaskAttemptID tid : tasksToKill) {
String msg = "Killing one of the least progress tasks - " + tid + ", as the cumulative memory usage of all the tasks on " + "the TaskTracker exceeds virtual memory limit " + maxMemoryAllowedForAllTasks + ".";
LOG.warn(msg);
killTask(tid, msg);
}
} else {
LOG.info("The total memory usage is overflowing TTs limits. " + "But found no alive task to kill for freeing memory.");
}
TODO What is compressible is turned on? LOG is a bad idea!
SATD_ADDED
write(DataOutput)
public void write(DataOutput out) throws IOException
// data bytes including vint encoding
WritableUtils.writeVInt(out, size);
final int payload = size - WritableUtils.getVIntSize(size);
if (payload > Long.SIZE / Byte.SIZE) {
if (compressible) {
writeRandomText(out, payload);
} else {
writeRandom(out, payload);
}
} else if (payload > 0) {
// TODO What is compressible is turned on? LOG is a bad idea!
out.write(literal, 0, payload);
}
TODO Should we use long for size. What if the data is more than 4G?
SATD_ADDED
writeRandomText(DataOutput, int)
private void writeRandomText(DataOutput out, final int size) throws IOException
long tmp = seed;
out.writeLong(tmp);
int i = size - (Long.SIZE / Byte.SIZE);
// TODO Should we use long for size. What if the data is more than 4G?
String randomWord = rtg.getRandomWord();
byte[] bytes = randomWord.getBytes("UTF-8");
long randomWordSize = bytes.length;
while (i >= randomWordSize) {
out.write(bytes);
i -= randomWordSize;
// get the next random word
randomWord = rtg.getRandomWord();
bytes = randomWord.getBytes("UTF-8");
// determine the random word size
randomWordSize = bytes.length;
}
// pad the remaining bytes
if (i > 0) {
out.write(bytes, 0, i);
}
create Configs.SYSTEM_DIR's parent with restrictive permissions.
So long as the JT has access to the system dir itself it should
be able to start.
SATD_ADDED
testGarbledMapredSystemDir()
public void testGarbledMapredSystemDir() throws Exception
Configuration conf = new Configuration();
final MiniDFSCluster dfs = new MiniDFSCluster(conf, 1, true, null);
MiniMRCluster mr = null;
try {
// start dfs
conf.set("dfs.permissions.supergroup", "supergroup");
FileSystem fs = DFS_UGI.doAs(new PrivilegedExceptionAction() {
public FileSystem run() throws IOException {
return dfs.getFileSystem();
}
});
// create Configs.SYSTEM_DIR's parent with restrictive permissions.
// So long as the JT has access to the system dir itself it should
// be able to start.
Path mapredSysDir = new Path(conf.get(JTConfig.JT_SYSTEM_DIR));
Path parentDir = mapredSysDir.getParent();
fs.mkdirs(parentDir);
fs.setPermission(parentDir, new FsPermission(SYSTEM_DIR_PARENT_PERMISSION));
fs.mkdirs(mapredSysDir);
fs.setPermission(mapredSysDir, new FsPermission(SYSTEM_DIR_PERMISSION));
fs.setOwner(mapredSysDir, "mr", "mrgroup");
// start mr (i.e jobtracker)
Configuration mrConf = new Configuration(conf);
mr = new MiniMRCluster(0, 0, 0, dfs.getFileSystem().getUri().toString(), 1, null, null, MR_UGI, new JobConf(mrConf));
JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
// add garbage to Configs.SYSTEM_DIR
Path garbage = new Path(jobtracker.getSystemDir(), "garbage");
fs.mkdirs(garbage);
fs.setPermission(garbage, new FsPermission(SYSTEM_DIR_PERMISSION));
fs.setOwner(garbage, "test", "test-group");
// stop the jobtracker
mr.stopJobTracker();
mr.getJobTrackerConf().setBoolean(JTConfig.JT_RESTART_ENABLED, false);
// start jobtracker but dont wait for it to be up
mr.startJobTracker(false);
// check 5 times .. each time wait for 2 secs to check if the jobtracker
// has crashed or not.
for (int i = 0; i < 5; ++i) {
LOG.info("Check #" + i);
if (!mr.getJobTrackerRunner().isActive()) {
return;
}
UtilsForTests.waitFor(2000);
}
assertFalse("JobTracker did not bail out (waited for 10 secs)", mr.getJobTrackerRunner().isActive());
} finally {
if (dfs != null) {
dfs.shutdown();
}
if (mr != null) {
mr.shutdown();
}
}
private Date longToDate(long val, int sqlDataType)
switch(sqlDataType) {
case Types.DATE:
return new java.sql.Date(val);
case Types.TIME:
return new java.sql.Time(val);
case Types.TIMESTAMP:
return new java.sql.Timestamp(val);
default:
// Shouldn't ever hit this case.
return null;
}
The range of acceptable dates is NULL to NULL. Just create a single split.
SATD_ADDED
split(Configuration, ResultSet, String)
public List split(Configuration conf, ResultSet results, String colName) throws SQLException
long minVal;
long maxVal;
int sqlDataType = results.getMetaData().getColumnType(1);
minVal = resultSetColToLong(results, 1, sqlDataType);
maxVal = resultSetColToLong(results, 2, sqlDataType);
String lowClausePrefix = colName + " >= ";
String highClausePrefix = colName + " < ";
int numSplits = conf.getInt(MRJobConfig.NUM_MAPS, 1);
if (numSplits < 1) {
numSplits = 1;
}
if (minVal == Long.MIN_VALUE && maxVal == Long.MIN_VALUE) {
// The range of acceptable dates is NULL to NULL. Just create a single split.
List splits = new ArrayList();
splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(colName + " IS NULL", colName + " IS NULL"));
return splits;
}
// Gather the split point integers
List splitPoints = split(numSplits, minVal, maxVal);
List splits = new ArrayList();
// Turn the split points into a set of intervals.
long start = splitPoints.get(0);
Date startDate = longToDate(start, sqlDataType);
if (sqlDataType == Types.TIMESTAMP) {
// The lower bound's nanos value needs to match the actual lower-bound nanos.
try {
((java.sql.Timestamp) startDate).setNanos(results.getTimestamp(1).getNanos());
} catch (NullPointerException npe) {
// If the lower bound was NULL, we'll get an NPE; just ignore it and don't set nanos.
}
}
for (int i = 1; i < splitPoints.size(); i++) {
long end = splitPoints.get(i);
Date endDate = longToDate(end, sqlDataType);
if (i == splitPoints.size() - 1) {
if (sqlDataType == Types.TIMESTAMP) {
// The upper bound's nanos value needs to match the actual upper-bound nanos.
try {
((java.sql.Timestamp) endDate).setNanos(results.getTimestamp(2).getNanos());
} catch (NullPointerException npe) {
// If the upper bound was NULL, we'll get an NPE; just ignore it and don't set nanos.
}
}
// This is the last one; use a closed interval.
splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(lowClausePrefix + dateToString(startDate), colName + " <= " + dateToString(endDate)));
} else {
// Normal open-interval case.
splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(lowClausePrefix + dateToString(startDate), highClausePrefix + dateToString(endDate)));
}
start = end;
startDate = endDate;
}
if (minVal == Long.MIN_VALUE || maxVal == Long.MIN_VALUE) {
// Add an extra split to handle the null case that we saw.
splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(colName + " IS NULL", colName + " IS NULL"));
}
return splits;
We need this odd-looking code because if a seed exists we need to ensure
that only one interpolator is generated per LoggedDiscreteCDF, but if no
seed exists then the potentially lengthy process of making an
interpolator can happen outside the lock. makeUpRuntimeCore only locks
around the two hash map accesses.
SATD_ADDED
convertState(Values)
private static State convertState(Values status)
if (status == Values.SUCCESS) {
return State.SUCCEEDED;
} else if (status == Values.FAILED) {
return State.FAILED;
} else if (status == Values.KILLED) {
return State.KILLED;
} else {
throw new IllegalArgumentException("unknown status " + status);
}
public void testDistributionPolicy() throws IOException
IndexUpdateConfiguration iconf = new IndexUpdateConfiguration(conf);
// test hashing distribution policy
iconf.setDistributionPolicyClass(HashingDistributionPolicy.class);
onetest();
if (fs.exists(indexPath)) {
fs.delete(indexPath, true);
}
// test round-robin distribution policy
iconf.setDistributionPolicyClass(RoundRobinDistributionPolicy.class);
onetest();
HACK ALERT!! It's possible for a Job end line to end a
job for which we have a config file
image [ a ParsedConfigFile ] in jobconf.
processParsedLine handles this.
SATD_ADDED
run()
Pair line = readBalancedLine();
while (line != null) {
if (debug && (lineNumber < 1000000L && lineNumber % 1000L == 0 || lineNumber % 1000000L == 0)) {
LOG.debug("" + lineNumber + " " + line.second());
}
if (line.first() == null) {
try {
// HACK ALERT!! It's possible for a Job end line to end a
// job for which we have a config file
// image [ a ParsedConfigFile ] in jobconf.
//
// processParsedLine handles this.
processParsedLine(new ParsedLine(line.second(), version));
} catch (StringIndexOutOfBoundsException e) {
LOG.warn("anomalous line #" + lineNumber + ":" + line, e);
}
} else {
jobconf = new ParsedConfigFile(line.first(), line.second());
if (jobconf.valid == false) {
jobconf = null;
}
maybeMateJobAndConf();
}
line = readBalancedLine();
}
finalizeJob();
if (collecting) {
String[] typeNames = LogRecordType.lineTypes();
for (int i = 0; i < typeNames.length; ++i) {
statisticalOutput.print(typeNames[i]);
statisticalOutput.print('\n');
}
} else {
if (delays) {
printDistributionSet("Job start delay spectrum:", delayTimeDists);
}
if (runtimes) {
printDistributionSet("Job run time spectrum:", runTimeDists);
}
if (spreading) {
String ratioDescription = "(" + spreadMax + "/1000 %ile) to (" + spreadMin + "/1000 %ile) scaled by 1000000";
printDistributionSet("Map task success times " + ratioDescription + ":", mapTimeSpreadDists);
printDistributionSet("Shuffle success times " + ratioDescription + ":", shuffleTimeSpreadDists);
printDistributionSet("Sort success times " + ratioDescription + ":", sortTimeSpreadDists);
printDistributionSet("Reduce success times " + ratioDescription + ":", reduceTimeSpreadDists);
}
if (collectTaskTimes) {
printDistributionSet("Global map task success times:", mapTimeDists);
printDistributionSet("Global shuffle task success times:", shuffleTimeDists);
printDistributionSet("Global sort task success times:", sortTimeDists);
printDistributionSet("Global reduce task success times:", reduceTimeDists);
}
}
if (topologyGen != null) {
LoggedNetworkTopology topo = new LoggedNetworkTopology(allHosts, "", 0);
topologyGen.writeObject(topo);
topologyGen.close();
}
if (jobTraceGen != null) {
jobTraceGen.close();
}
if (input != null) {
input.close();
input = null;
}
if (inputCodec != null) {
CodecPool.returnDecompressor(inputDecompressor);
inputDecompressor = null;
inputCodec = null;
}
return 0;
we first start a cluster and fill the cluster up to a certain size.
then redistribute blocks according the required distribution.
Afterwards a balancer is running to balance the cluster.
int numDatanodes = distribution.length;
if (capacities.length != numDatanodes || racks.length != numDatanodes) {
throw new IllegalArgumentException("Array length is not the same");
}
// calculate total space that need to be filled
final long totalUsedSpace = sum(distribution);
// fill the cluster
ExtendedBlock[] blocks = generateBlocks(conf, totalUsedSpace, (short) numDatanodes);
// redistribute blocks
Block[][] blocksDN = distributeBlocks(blocks, (short) (numDatanodes - 1), distribution);
// restart the cluster: do NOT format the cluster
conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "0.0f");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).racks(racks).simulatedCapacities(capacities).build();
cluster.waitActive();
client = DFSClient.createNamenode(conf);
for (int i = 0; i < blocksDN.length; i++) cluster.injectBlocks(i, Arrays.asList(blocksDN[i]));
final long totalCapacity = sum(capacities);
runBalancer(conf, totalUsedSpace, totalCapacity);
cluster.shutdown();
String id = file + timeStamp;
if (!isPublic) {
// consider job-submitter for private distributed cache file
id = id.concat(user);
}
return new Path(distCachePath, MD5Hash.digest(id).toString()).toUri().getPath();
There is high probability that the rack id generated here will
not conflict with those of other data node cluster.
Not perfect but mostly unique rack ids are good enough
SATD_ADDED
getUniqueRackPrefix()
private static String getUniqueRackPrefix()
String ip = "unknownIP";
try {
ip = DNS.getDefaultIP("default");
} catch (UnknownHostException ignored) {
System.out.println("Could not find ip address of \"default\" inteface.");
}
int rand = 0;
try {
rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE);
} catch (NoSuchAlgorithmException e) {
rand = (new Random()).nextInt(Integer.MAX_VALUE);
}
return "/Rack-" + rand + "-" + ip + "-" + System.currentTimeMillis();
Refresh the service level authorization policy once again,
this time it should fail!
SATD_ADDED
testRefresh()
public void testRefresh() throws Exception
MiniDFSCluster dfs = null;
try {
final int slaves = 4;
// Turn on service-level authorization
final Configuration conf = new Configuration();
conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class);
conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true);
// Start the mini dfs cluster
dfs = new MiniDFSCluster(conf, slaves, true, null);
// Refresh the service level authorization policy
refreshPolicy(conf);
// Simulate an 'edit' of hadoop-policy.xml
String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf");
String HADOOP_POLICY_FILE = System.getProperty("hadoop.policy.file");
File policyFile = new File(confDir, HADOOP_POLICY_FILE);
String policyFileCopy = HADOOP_POLICY_FILE + ".orig";
// first save original
FileUtil.copy(// first save original
policyFile, // first save original
FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf);
rewriteHadoopPolicyFile(new File(confDir, HADOOP_POLICY_FILE));
// Refresh the service level authorization policy
refreshPolicy(conf);
// Refresh the service level authorization policy once again,
// this time it should fail!
try {
// Note: hadoop-policy.xml for tests has
// security.refresh.policy.protocol.acl = ${user.name}
UserGroupInformation unknownUser = UserGroupInformation.createRemoteUser("unknown");
unknownUser.doAs(new PrivilegedExceptionAction() {
public Void run() throws IOException {
refreshPolicy(conf);
return null;
}
});
fail("Refresh of NameNode's policy file cannot be successful!");
} catch (Exception re) {
System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re));
} finally {
// Reset to original hadoop-policy.xml
FileUtil.fullyDelete(new File(confDir, HADOOP_POLICY_FILE));
FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, HADOOP_POLICY_FILE));
}
} finally {
if (dfs != null) {
dfs.shutdown();
}
}
MiniDFSCluster dfs = null;
try {
final int slaves = 4;
// Turn on service-level authorization
final Configuration conf = new Configuration();
conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class);
conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true);
// Start the mini dfs cluster
dfs = new MiniDFSCluster(conf, slaves, true, null);
// Refresh the service level authorization policy
refreshPolicy(conf);
// Simulate an 'edit' of hadoop-policy.xml
String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf");
String HADOOP_POLICY_FILE = System.getProperty("hadoop.policy.file");
File policyFile = new File(confDir, HADOOP_POLICY_FILE);
String policyFileCopy = HADOOP_POLICY_FILE + ".orig";
// first save original
FileUtil.copy(// first save original
policyFile, // first save original
FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf);
rewriteHadoopPolicyFile(new File(confDir, HADOOP_POLICY_FILE));
// Refresh the service level authorization policy
refreshPolicy(conf);
// Refresh the service level authorization policy once again,
// this time it should fail!
try {
// Note: hadoop-policy.xml for tests has
// security.refresh.policy.protocol.acl = ${user.name}
UserGroupInformation unknownUser = UserGroupInformation.createRemoteUser("unknown");
unknownUser.doAs(new PrivilegedExceptionAction() {
public Void run() throws IOException {
refreshPolicy(conf);
return null;
}
});
fail("Refresh of NameNode's policy file cannot be successful!");
} catch (Exception re) {
System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re));
} finally {
// Reset to original hadoop-policy.xml
FileUtil.fullyDelete(new File(confDir, HADOOP_POLICY_FILE));
FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, HADOOP_POLICY_FILE));
}
} finally {
if (dfs != null) {
dfs.shutdown();
}
}
Stop and start the tracker. The tracker should come up nicely
SATD_ADDED
testHistoryInitWithCorruptFiles()
public void testHistoryInitWithCorruptFiles() throws IOException
MiniMRCluster mr = null;
try {
JobConf conf = new JobConf();
Path historyDir = new Path(System.getProperty("test.build.data", "."), "history");
conf.set(JTConfig.JT_JOBHISTORY_LOCATION, historyDir.toString());
conf.setUser("user");
FileSystem localFs = FileSystem.getLocal(conf);
// there may be some stale files, clean them
if (localFs.exists(historyDir)) {
boolean deleted = localFs.delete(historyDir, true);
LOG.info(historyDir + " deleted " + deleted);
}
// Start the cluster, create a history file
mr = new MiniMRCluster(0, "file:///", 3, null, null, conf);
JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
JobHistory jh = jt.getJobHistory();
final JobID jobId = JobID.forName("job_200809171136_0001");
jh.setupEventWriter(jobId, conf);
Map jobACLs = new HashMap();
JobSubmittedEvent jse = new JobSubmittedEvent(jobId, "job", "user", 12345, "path", jobACLs, "default");
jh.logEvent(jse, jobId);
jh.closeWriter(jobId);
// Corrupt the history file. User RawLocalFileSystem so that we
// do keep the original CRC file intact.
String historyFileName = jobId.toString() + "_" + "user";
Path historyFilePath = new Path(historyDir.toString(), historyFileName);
RawLocalFileSystem fs = (RawLocalFileSystem) FileSystem.getLocal(conf).getRaw();
FSDataOutputStream out = fs.create(historyFilePath, true);
byte[] corruptData = new byte[32];
new Random().nextBytes(corruptData);
out.write(corruptData, 0, 32);
out.close();
// Stop and start the tracker. The tracker should come up nicely
mr.stopJobTracker();
mr.startJobTracker();
jt = mr.getJobTrackerRunner().getJobTracker();
assertNotNull("JobTracker did not come up", jt);
jh = jt.getJobHistory();
assertNotNull("JobHistory did not get initialized correctly", jh);
// Only the done folder should remain in the history directory
assertEquals("Files in logDir did not move to DONE folder", 1, historyDir.getFileSystem(conf).listStatus(historyDir).length);
} finally {
if (mr != null) {
cleanupLocalFiles(mr);
mr.shutdown();
}
}
String[] parts = path.getName().split("_");
// TODO this is a hack :(
// jobtracker-hostname_jobtracker-identifier_
String id = parts[0] + "_" + parts[1] + "_" + parts[2];
return new Path(dir, id + "_conf.xml");
public void testHighMemoryBlockingAcrossTaskTypes() throws IOException
// 2 map and 1 reduce slots
taskTrackerManager = new FakeTaskTrackerManager(1, 2, 2);
taskTrackerManager.addQueues(new String[] { "default" });
ArrayList queues = new ArrayList();
queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
scheduler.setTaskTrackerManager(taskTrackerManager);
// enabled memory-based scheduling
// Normal job in the cluster would be 1GB maps/reduces
scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024);
scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 1 * 1024);
scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024);
taskTrackerManager.setFakeQueues(queues);
scheduler.start();
// The situation : Two jobs in the queue. First job with only maps and no
// reduces and is a high memory job. Second job is a normal job with both
// maps and reduces.
// First job cannot run for want of memory for maps. In this case, second
// job's reduces should run.
LOG.debug("Submit one high memory(2GB maps, 0MB reduces) job of " + "2 map tasks");
JobConf jConf = new JobConf(conf);
jConf.setMemoryForMapTask(2 * 1024);
jConf.setMemoryForReduceTask(0);
jConf.setNumMapTasks(2);
jConf.setNumReduceTasks(0);
jConf.setQueueName("default");
jConf.setUser("u1");
FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf);
LOG.debug("Submit another regular memory(1GB vmem maps/reduces) job of " + "2 map/red tasks");
jConf = new JobConf(conf);
jConf.setMemoryForMapTask(1 * 1024);
jConf.setMemoryForReduceTask(1 * 1024);
jConf.setNumMapTasks(2);
jConf.setNumReduceTasks(2);
jConf.setQueueName("default");
jConf.setUser("u1");
FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf);
// first, a map from j1 and a reduce from other job j2
Map strs = new HashMap();
strs.put(MAP, "attempt_test_0001_m_000001_0 on tt1");
strs.put(REDUCE, "attempt_test_0002_r_000001_0 on tt1");
checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", strs);
// Total 2 map slots should be accounted for.
checkOccupiedSlots("default", TaskType.MAP, 1, 2, 100.0f);
checkOccupiedSlots("default", TaskType.REDUCE, 1, 1, 50.0f);
checkMemReservedForTasksOnTT("tt1", 2 * 1024L, 1 * 1024L);
// TT has 2 slots for reduces hence this call should get a reduce task
// from other job
checkAssignment(taskTrackerManager, scheduler, "tt1", "attempt_test_0002_r_000002_0 on tt1");
checkOccupiedSlots("default", TaskType.MAP, 1, 2, 100.0f);
checkOccupiedSlots("default", TaskType.REDUCE, 1, 2, 100.0f);
checkMemReservedForTasksOnTT("tt1", 2 * 1024L, 2 * 1024L);
// now as all the slots are occupied hence no more tasks would be
// assigned.
assertNull(scheduler.assignTasks(tracker("tt1")));
u1 has 5 map slots and 5 reduce slots. u2 has 4 map slots and 4 reduce
slots. Because of high memory tasks, giving u2 another task would
overflow limits. So, no more tasks should be given to anyone.
SATD_ADDED
testUserLimitsForHighMemoryJobs()
public void testUserLimitsForHighMemoryJobs() throws IOException
taskTrackerManager = new FakeTaskTrackerManager(1, 10, 10);
scheduler.setTaskTrackerManager(taskTrackerManager);
String[] qs = { "default" };
taskTrackerManager.addQueues(qs);
ArrayList queues = new ArrayList();
queues.add(new FakeQueueInfo("default", 100.0f, true, 50));
// enabled memory-based scheduling
// Normal job in the cluster would be 1GB maps/reduces
scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024);
scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 2 * 1024);
scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024);
taskTrackerManager.setFakeQueues(queues);
scheduler.start();
// Submit one normal job to the other queue.
JobConf jConf = new JobConf(conf);
jConf.setMemoryForMapTask(1 * 1024);
jConf.setMemoryForReduceTask(1 * 1024);
jConf.setNumMapTasks(6);
jConf.setNumReduceTasks(6);
jConf.setUser("u1");
jConf.setQueueName("default");
FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf);
LOG.debug("Submit one high memory(2GB maps, 2GB reduces) job of " + "6 map and 6 reduce tasks");
jConf = new JobConf(conf);
jConf.setMemoryForMapTask(2 * 1024);
jConf.setMemoryForReduceTask(2 * 1024);
jConf.setNumMapTasks(6);
jConf.setNumReduceTasks(6);
jConf.setQueueName("default");
jConf.setUser("u2");
FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf);
// Verify that normal job takes 5 task assignments to hit user limits
Map expectedStrings = new HashMap();
for (int i = 0; i < 5; i++) {
expectedStrings.clear();
expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0001_m_00000" + (i + 1) + "_0 on tt1");
expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0001_r_00000" + (i + 1) + "_0 on tt1");
checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", expectedStrings);
}
// u1 has 5 map slots and 5 reduce slots. u2 has none. So u1's user limits
// are hit. So u2 should get slots
for (int i = 0; i < 2; i++) {
expectedStrings.clear();
expectedStrings.put(CapacityTestUtils.MAP, "attempt_test_0002_m_00000" + (i + 1) + "_0 on tt1");
expectedStrings.put(CapacityTestUtils.REDUCE, "attempt_test_0002_r_00000" + (i + 1) + "_0 on tt1");
checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", expectedStrings);
}
// u1 has 5 map slots and 5 reduce slots. u2 has 4 map slots and 4 reduce
// slots. Because of high memory tasks, giving u2 another task would
// overflow limits. So, no more tasks should be given to anyone.
assertNull(scheduler.assignTasks(tracker("tt1")));
by doing the above clock adjustments, we bring the progress rate of
taskID 3 lower than 4. For taskID 3, the rate is 85/317000
and for taskID 4, the rate is 20/65000. But when we ask for a spec task
now, we should get back taskID 4 (since that is expected to complete
later than taskID 3).
SATD_ADDED
testTaskLATEScheduling()
public void testTaskLATEScheduling() throws IOException
TaskAttemptID[] taskAttemptID = new TaskAttemptID[20];
JobConf conf = new JobConf();
conf.setSpeculativeExecution(true);
conf.setNumMapTasks(5);
conf.setNumReduceTasks(0);
conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
job.initTasks();
taskAttemptID[0] = job.findMapTask(trackers[0]);
taskAttemptID[1] = job.findMapTask(trackers[1]);
taskAttemptID[2] = job.findMapTask(trackers[2]);
taskAttemptID[3] = job.findMapTask(trackers[3]);
clock.advance(2000);
job.finishTask(taskAttemptID[0]);
job.finishTask(taskAttemptID[1]);
job.finishTask(taskAttemptID[2]);
clock.advance(250000);
taskAttemptID[4] = job.findMapTask(trackers[3]);
clock.advanceBySpeculativeLag();
// by doing the above clock adjustments, we bring the progress rate of
// taskID 3 lower than 4. For taskID 3, the rate is 85/317000
// and for taskID 4, the rate is 20/65000. But when we ask for a spec task
// now, we should get back taskID 4 (since that is expected to complete
// later than taskID 3).
job.progressMade(taskAttemptID[3], 0.85f);
job.progressMade(taskAttemptID[4], 0.20f);
taskAttemptID[5] = job.findMapTask(trackers[4]);
assertEquals(taskAttemptID[5].getTaskID().getId(), 4);
// Verify total speculative tasks by jobtracker instrumentation
assertEquals("Total speculative maps", 2, fakeInst.numSpeculativeMaps);
assertEquals("Total speculative reduces", 3, fakeInst.numSpeculativeReduces);
LOG.info("Total speculative maps = " + fakeInst.numSpeculativeMaps);
LOG.info("Total speculative reduces = " + fakeInst.numSpeculativeReduces);
TODO Control the extra data written ..
TODO Should the key\tvalue\n be considered for measuring size?
Can counters like BYTES_WRITTEN be used? What will be the value of
such counters in LocalJobRunner?
// Ask the user which local directory to upload
DirectoryDialog dialog = new DirectoryDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI);
dialog.setText("Select the local file or directory to upload");
String dirName = dialog.open();
final File dir = new File(dirName);
List files = new ArrayList();
files.add(dir);
// TODO enable upload command only when selection is exactly one folder
final List folders = filterSelection(DFSFolder.class, selection);
if (folders.size() >= 1)
uploadToDFS(folders.get(0), files);
// Ask the user which files to upload
FileDialog dialog = new FileDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI);
dialog.setText("Select the local files to upload");
dialog.open();
List files = new ArrayList();
for (String fname : dialog.getFileNames()) files.add(new File(dialog.getFilterPath() + File.separator + fname));
// TODO enable upload command only when selection is exactly one folder
List folders = filterSelection(DFSFolder.class, selection);
if (folders.size() >= 1)
uploadToDFS(folders.get(0), files);
private void mySetup(String erasureCode, int rsParityLength) throws Exception
// Make sure data directory exists
new File(TEST_DIR).mkdirs();
conf = new Configuration();
conf.set("fs.raid.recoverylogdir", LOG_DIR);
conf.setInt(RaidNode.RS_PARITY_LENGTH_KEY, rsParityLength);
// scan all policies once every 5 second
conf.setLong("raid.policy.rescan.interval", 5000);
// make all deletions not go through Trash
conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
// do not use map-reduce cluster for Raiding
conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
conf.set("raid.server.address", "localhost:0");
conf.setInt("hdfs.raid.stripeLength", stripeLength);
conf.set("xor".equals(erasureCode) ? RaidNode.RAID_LOCATION_KEY : RaidNode.RAIDRS_LOCATION_KEY, "/destraid");
dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
dfs.waitActive();
fileSys = dfs.getFileSystem();
namenode = fileSys.getUri().toString();
hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
FileSystem.setDefaultUri(conf, namenode);
explicitly do not use \"normal\" job.setOutputPath to make sure
that it is not hardcoded anywhere in the framework.
SATD_ADDED
testWithLocal()
public void testWithLocal() throws IOException, InterruptedException, ClassNotFoundException
MiniMRCluster mr = null;
try {
mr = new MiniMRCluster(2, "file:///", 3);
// make cleanup inline sothat validation of existence of these directories
// can be done
mr.setInlineCleanupThreads();
TestMiniMRWithDFS.runPI(mr, mr.createJobConf());
// run the wordcount example with caching
JobConf job = mr.createJobConf();
TestResult ret = MRCaching.launchMRCache(TEST_ROOT_DIR + "/wc/input", TEST_ROOT_DIR + "/wc/output", TEST_ROOT_DIR + "/cachedir", job, "The quick brown fox\n" + "has many silly\n" + "red fox sox\n");
// assert the number of lines read during caching
assertTrue("Failed test archives not matching", ret.isOutputOk);
// test the task report fetchers
JobClient client = new JobClient(job);
JobID jobid = ret.job.getID();
TaskReport[] reports;
reports = client.getSetupTaskReports(jobid);
assertEquals("number of setups", 2, reports.length);
reports = client.getMapTaskReports(jobid);
assertEquals("number of maps", 1, reports.length);
reports = client.getReduceTaskReports(jobid);
assertEquals("number of reduces", 1, reports.length);
reports = client.getCleanupTaskReports(jobid);
assertEquals("number of cleanups", 2, reports.length);
Counters counters = ret.job.getCounters();
assertEquals("number of map inputs", 3, counters.getCounter(TaskCounter.MAP_INPUT_RECORDS));
assertEquals("number of reduce outputs", 9, counters.getCounter(TaskCounter.REDUCE_OUTPUT_RECORDS));
runCustomFormats(mr);
runSecondarySort(mr.createJobConf());
} finally {
if (mr != null) {
mr.shutdown();
}
}
if remainder is between max and 2*max - then
instead of creating splits of size max, left-max we
create splits of size left/2 and left/2. This is
a heuristic to avoid creating really really small
splits.
Create one split for this rack before moving over to the next rack.
Come back to this rack after creating a single split for each of the
remaining racks.
Process one rack location at a time, Combine all possible blocks that
reside on this rack as one split. (constrained by minimum and maximum
split size).
Hack for local FS that does not have the concept of a 'mounting point'
SATD_ADDED
getFlagDir(boolean)
private static Path getFlagDir(boolean local)
Path flagDir = new Path("testing/chain/flags");
// Hack for local FS that does not have the concept of a 'mounting point'
if (local) {
String localPathRoot = System.getProperty("test.build.data", "/tmp").replace(' ', '+');
flagDir = new Path(localPathRoot, flagDir);
}
return flagDir;
for security authorization
server principal for this call
should be NAMENODE's one.
SATD_ADDED
refreshSuperUserGroupsConfiguration()
public int refreshSuperUserGroupsConfiguration() throws IOException
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be NAMENODE's one.
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
// Create the client
RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class));
// Refresh the user-to-groups mappings
refreshProtocol.refreshSuperUserGroupsConfiguration();
return 0;
for security authorization
server principal for this call
should be NN's one.
SATD_ADDED
refreshUserToGroupsMappings()
public int refreshUserToGroupsMappings() throws IOException
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be NN's one.
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
// Create the client
RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class));
// Refresh the user-to-groups mappings
refreshProtocol.refreshUserToGroupsMappings();
return 0;
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be NN's one.
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
// Create the client
RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class));
// Refresh the authorization policy in-effect
refreshProtocol.refreshServiceAcl();
return 0;
for security authorization
server principal for this call
should be NN's one.
SATD_ADDED
refreshServiceAcl()
public int refreshServiceAcl() throws IOException
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be NN's one.
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
// Create the client
RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, NameNode.getAddress(conf), getUGI(), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class));
// Refresh the authorization policy in-effect
refreshProtocol.refreshServiceAcl();
return 0;
it's fine walking down the entire list of running jobs - there
probably will not be many, plus, we may need to go through the
list to compute numSlotsOccupiedByUser. If this is expensive, we
can keep a list of running jobs per user. Then we only need to
consider the first few jobs per user.
SATD_ADDED
compare(JobSchedulingInfo, JobSchedulingInfo)
public int compare(JobSchedulingInfo o1, JobSchedulingInfo o2)
// the job that started earlier wins
if (o1.getStartTime() < o2.getStartTime()) {
return -1;
} else {
return (o1.getStartTime() == o2.getStartTime() ? o1.getJobID().compareTo(o2.getJobID()) : 1);
}
This kind of error doesn't mean that the stream itself is broken - just the
flushing thread got interrupted. So, we shouldn't close down the writer,
but instead just propagate the error
SATD_ADDED
hflush()
public void hflush() throws IOException
dfsClient.checkOpen();
isClosed();
try {
long toWaitFor;
synchronized (this) {
/* Record current blockOffset. This might be changed inside
* flushBuffer() where a partial checksum chunk might be flushed.
* After the flush, reset the bytesCurBlock back to its previous value,
* any partial checksum chunk will be sent now and in next packet.
*/
long saveOffset = bytesCurBlock;
Packet oldCurrentPacket = currentPacket;
// flush checksum buffer, but keep checksum buffer intact
flushBuffer(true);
// bytesCurBlock potentially incremented if there was buffered data
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("DFSClient flush() : saveOffset " + saveOffset + " bytesCurBlock " + bytesCurBlock + " lastFlushOffset " + lastFlushOffset);
}
// Flush only if we haven't already flushed till this offset.
if (lastFlushOffset != bytesCurBlock) {
assert bytesCurBlock > lastFlushOffset;
// record the valid offset of this flush
lastFlushOffset = bytesCurBlock;
waitAndQueueCurrentPacket();
} else {
// We already flushed up to this offset.
// This means that we haven't written anything since the last flush
// (or the beginning of the file). Hence, we should not have any
// packet queued prior to this call, since the last flush set
// currentPacket = null.
assert oldCurrentPacket == null : "Empty flush should not occur with a currentPacket";
// just discard the current packet since it is already been sent.
currentPacket = null;
}
// Restore state of stream. Record the last flush offset
// of the last full chunk that was flushed.
//
bytesCurBlock = saveOffset;
toWaitFor = lastQueuedSeqno;
}
// end synchronized
waitForAckedSeqno(toWaitFor);
// If any new blocks were allocated since the last flush,
// then persist block locations on namenode.
//
if (persistBlocks.getAndSet(false)) {
try {
dfsClient.namenode.fsync(src, dfsClient.clientName);
} catch (IOException ioe) {
DFSClient.LOG.warn("Unable to persist blocks in hflush for " + src, ioe);
// If we got an error here, it might be because some other thread called
// close before our hflush completed. In that case, we should throw an
// exception that the stream is closed.
isClosed();
// If we aren't closed but failed to sync, we should expose that to the
// caller.
throw ioe;
}
}
synchronized (this) {
if (streamer != null) {
streamer.setHflush();
}
}
} catch (InterruptedIOException interrupt) {
// This kind of error doesn't mean that the stream itself is broken - just the
// flushing thread got interrupted. So, we shouldn't close down the writer,
// but instead just propagate the error
throw interrupt;
} catch (IOException e) {
DFSClient.LOG.warn("Error while syncing", e);
synchronized (this) {
if (!closed) {
lastException = new IOException("IOException flush:" + e);
closeThreads(true);
}
}
throw e;
}
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
The sleeping period before checking if block move is completed again
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
Start a thread to dispatch block moves for each source.
The thread selects blocks to move & sends request to proxy source to
initiate block move. The process is flow controlled. Block selection is
blocked if there are too many un-confirmed block moves.
Return the total number of bytes successfully moved in this iteration.
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
Decide all pairs and
the number of bytes to move from a source to a target
Maximum bytes to be moved per node is
Min(1 Band worth of bytes, MAX_SIZE_TO_MOVE).
Return total number of bytes to move in this iteration
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
This method iteratively does the following:
it first selects a block to move,
then sends a request to the proxy source to start the block move
when the source's block list falls below a threshold, it asks
the namenode for more blocks.
It terminates when it has dispatch enough block move tasks or
it has received enough blocks from the namenode, or
the elapsed time of the iteration has exceeded the max time limit.
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
Return a block that's good for the source thread to dispatch immediately
The block's source, target, and proxy source are determined too.
When choosing proxy and target, source & target throttling
has been considered. They are chosen only when they have the capacity
to support this block move.
The block should be dispatched immediately after this method is returned.
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
Decide if the given block is a good candidate to move or not
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
A thread that initiates a block move
and waits for block move to complete
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
Dispatch the block move task to the proxy source & wait for the response
SATD_ADDED
chooseBlockAndProxy()
private boolean chooseBlockAndProxy()
// iterate all source's blocks until find a good one
for (Iterator blocks = source.getBlockIterator(); blocks.hasNext(); ) {
if (markMovedIfGoodBlock(blocks.next())) {
blocks.remove();
return true;
}
}
return false;
TODO The only problem with these counters would be on restart.
The jobtracker updates the counter only when the task that is scheduled
if from a non-running tip and is local (data, rack ...). But upon restart
as the reports come from the task tracker, there is no good way to infer
when exactly to increment the locality counters. The only solution is to
increment the counters for all the tasks irrespective of
- whether the tip is running or not
- whether its a speculative task or not
So to simplify, increment the data locality counter whenever there is
data locality.
TODO Remove this once LocalJobRunner can run Gridmix.
SATD_ADDED
getSplits(JobContext)
public List getSplits(JobContext jobCtxt) throws IOException
// get the total data to be generated
long toGen = jobCtxt.getConfiguration().getLong(GenerateData.GRIDMIX_GEN_BYTES, -1);
if (toGen < 0) {
throw new IOException("Invalid/missing generation bytes: " + toGen);
}
// get the total number of mappers configured
int totalMappersConfigured = jobCtxt.getConfiguration().getInt(MRJobConfig.NUM_MAPS, -1);
if (totalMappersConfigured < 0) {
throw new IOException("Invalid/missing num mappers: " + totalMappersConfigured);
}
final long bytesPerTracker = toGen / totalMappersConfigured;
final ArrayList splits = new ArrayList(totalMappersConfigured);
for (int i = 0; i < totalMappersConfigured; ++i) {
splits.add(new GenSplit(bytesPerTracker, new String[] { "tracker_local" }));
}
return splits;
rare case where splits are exact, logs.length can be 4
SATD_ADDED
reset()
void reset()
final int oldsize = size;
do {
size = gen.nextInt(MAX_SIZE);
} while (oldsize == size);
final long oldseed = seed;
do {
seed = gen.nextLong() & Long.MAX_VALUE;
} while (oldseed == seed);
add a block to a under replication queue according to its priority
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
SATD_ADDED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
Return the priority of a block
@param block a under replication block
@param curReplicas current number of replicas of the block
@param expectedReplicas expected number of replicas of the block
SATD_ADDED
clear()
void clear()
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
Don't check for exact values in the middle, because the splitter generates some
ugly Unicode-isms. But do check that we get multiple splits and that it starts
and ends on the correct points.
SATD_ADDED
testCommonPrefix()
public void testCommonPrefix() throws SQLException
// Splits between 'Hand' and 'Hardy'
TextSplitter splitter = new TextSplitter();
List splits = splitter.split(5, "nd", "rdy", "Ha");
// Don't check for exact values in the middle, because the splitter generates some
// ugly Unicode-isms. But do check that we get multiple splits and that it starts
// and ends on the correct points.
assertEquals("Hand", splits.get(0));
assertEquals("Hardy", splits.get(splits.size() - 1));
assertEquals(6, splits.size());
// decrement number of blocks scheduled to this datanode.
node.decBlocksScheduled();
// get the deletion hint node
DatanodeDescriptor delHintNode = null;
if (delHint != null && delHint.length() != 0) {
delHintNode = namesystem.getDatanode(delHint);
if (delHintNode == null) {
NameNode.stateChangeLog.warn("BLOCK* NameSystem.blockReceived: " + block + " is expected to be removed from an unrecorded node " + delHint);
}
}
//
// Modify the blocks->datanode map and node's map.
//
pendingReplications.remove(block);
// blockReceived reports a finalized block
Collection toAdd = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
processReportedBlock(node, block, ReplicaState.FINALIZED, toAdd, toInvalidate, toCorrupt, toUC);
// the block is only in one of the to-do lists
// if it is in none then data-node already has it
assert toUC.size() + toAdd.size() + toInvalidate.size() + toCorrupt.size() <= 1 : "The block should be only in one of the lists.";
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, delHintNode, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.addBlock: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
The next two methods test the various cases under which we must conclude
the replica is corrupt, or under construction. These are laid out
as switch statements, on the theory that it is easier to understand
the combinatorics of reportedState and ucState that way. It should be
at least as efficient as boolean expressions.
switch(reportedState) {
case FINALIZED:
switch(ucState) {
case COMPLETE:
case COMMITTED:
return (storedBlock.getGenerationStamp() != iblk.getGenerationStamp() || storedBlock.getNumBytes() != iblk.getNumBytes());
default:
return false;
}
case RBW:
case RWR:
return storedBlock.isComplete();
// should not be reported
case RUR:
// should not be reported
case TEMPORARY:
default:
FSNamesystem.LOG.warn("Unexpected replica state " + reportedState + " for block: " + storedBlock + " on " + dn.getName() + " size " + storedBlock.getNumBytes());
return true;
}
// add to under-construction list
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
BlockInfo delimiter = new BlockInfo(new Block(), 1);
boolean added = dn.addBlock(delimiter);
assert added : "Delimiting block cannot be present in the node";
if (newReport == null)
newReport = new BlockListAsLongs();
// scan the report and process newly reported blocks
BlockReportIterator itBR = newReport.getBlockReportIterator();
while (itBR.hasNext()) {
Block iblk = itBR.next();
ReplicaState iState = itBR.getCurrentReplicaState();
BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
if (storedBlock != null && storedBlock.findDatanode(dn) >= 0)
dn.moveBlockToHead(storedBlock);
}
// collect blocks that have not been reported
// all of them are next to the delimiter
Iterator it = new DatanodeDescriptor.BlockIterator(delimiter.getNext(0), dn);
while (it.hasNext()) toRemove.add(it.next());
dn.removeBlock(delimiter);
public void processReport(DatanodeDescriptor node, BlockListAsLongs report) throws IOException
boolean isFirstBlockReport = (node.numBlocks() == 0);
if (isFirstBlockReport) {
// Initial block reports can be processed a lot more efficiently than
// ordinary block reports. This shortens NN restart times.
processFirstBlockReport(node, report);
return;
}
// Normal case:
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
Collection toAdd = new LinkedList();
Collection toRemove = new LinkedList();
Collection toInvalidate = new LinkedList();
Collection toCorrupt = new LinkedList();
Collection toUC = new LinkedList();
reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC);
// Process the blocks on each queue
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState);
}
for (Block b : toRemove) {
removeStoredBlock(b, node);
}
for (BlockInfo b : toAdd) {
addStoredBlock(b, node, null, true);
}
for (Block b : toInvalidate) {
NameNode.stateChangeLog.info("BLOCK* NameSystem.processReport: block " + b + " on " + node.getName() + " size " + b.getNumBytes() + " does not belong to any file.");
addToInvalidates(b, node);
}
for (BlockInfo b : toCorrupt) {
markBlockAsCorrupt(b, node);
}
TODO: ensure that no jobs/tasks are running
restart the cluster if cleanup fails
SATD_ADDED
ensureClean()
public void ensureClean() throws IOException
// TODO: ensure that no jobs/tasks are running
// restart the cluster if cleanup fails
JTClient jtClient = getJTClient();
JobInfo[] jobs = jtClient.getProxy().getAllJobInfo();
for (JobInfo job : jobs) {
jtClient.killJob(org.apache.hadoop.mapred.JobID.downgrade(job.getID()));
}
Now we need to generate the random numbers according to
the above distribution.
We create a lot of map tasks, each of which takes at least
one \"line\" of the distribution. (That is, a certain number
X is to be generated Y number of times.)
A map task emits Y key/val pairs. The val is X. The key
is a randomly-generated number.
The reduce task gets its input sorted by key. That is, sorted
in random order. It then emits a single line of text that
for the given values. It does not emit the key.
Because there's just one reduce task, we emit a single big
file of random numbers.
SATD_ADDED
launch()
private static void launch() throws Exception
//
// Generate distribution of ints. This is the answer key.
//
Configuration conf = new Configuration();
int countsToGo = counts;
int[] dist = new int[range];
for (int i = 0; i < range; i++) {
double avgInts = (1.0 * countsToGo) / (range - i);
dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian())));
countsToGo -= dist[i];
}
if (countsToGo > 0) {
dist[dist.length - 1] += countsToGo;
}
//
// Write the answer key to a file.
//
Path testdir = new Path("mapred.loadtest");
if (!fs.mkdirs(testdir)) {
throw new IOException("Mkdirs failed to create " + testdir.toString());
}
Path randomIns = new Path(testdir, "genins");
if (!fs.mkdirs(randomIns)) {
throw new IOException("Mkdirs failed to create " + randomIns.toString());
}
Path answerkey = new Path(randomIns, "answer.key");
SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, answerkey, IntWritable.class, IntWritable.class, SequenceFile.CompressionType.NONE);
try {
for (int i = 0; i < range; i++) {
out.append(new IntWritable(i), new IntWritable(dist[i]));
}
} finally {
out.close();
}
printFiles(randomIns, conf);
//
// Now we need to generate the random numbers according to
// the above distribution.
//
// We create a lot of map tasks, each of which takes at least
// one "line" of the distribution. (That is, a certain number
// X is to be generated Y number of times.)
//
// A map task emits Y key/val pairs. The val is X. The key
// is a randomly-generated number.
//
// The reduce task gets its input sorted by key. That is, sorted
// in random order. It then emits a single line of text that
// for the given values. It does not emit the key.
//
// Because there's just one reduce task, we emit a single big
// file of random numbers.
//
Path randomOuts = new Path(testdir, "genouts");
fs.delete(randomOuts, true);
Job genJob = Job.getInstance(conf);
FileInputFormat.setInputPaths(genJob, randomIns);
genJob.setInputFormatClass(SequenceFileInputFormat.class);
genJob.setMapperClass(RandomGenMapper.class);
FileOutputFormat.setOutputPath(genJob, randomOuts);
genJob.setOutputKeyClass(IntWritable.class);
genJob.setOutputValueClass(IntWritable.class);
genJob.setReducerClass(RandomGenReducer.class);
genJob.setNumReduceTasks(1);
genJob.waitForCompletion(true);
printFiles(randomOuts, conf);
//
// Next, we read the big file in and regenerate the
// original map. It's split into a number of parts.
// (That number is 'intermediateReduces'.)
//
// We have many map tasks, each of which read at least one
// of the output numbers. For each number read in, the
// map task emits a key/value pair where the key is the
// number and the value is "1".
//
// We have a single reduce task, which receives its input
// sorted by the key emitted above. For each key, there will
// be a certain number of "1" values. The reduce task sums
// these values to compute how many times the given key was
// emitted.
//
// The reduce task then emits a key/val pair where the key
// is the number in question, and the value is the number of
// times the key was emitted. This is the same format as the
// original answer key (except that numbers emitted zero times
// will not appear in the regenerated key.) The answer set
// is split into a number of pieces. A final MapReduce job
// will merge them.
//
// There's not really a need to go to 10 reduces here
// instead of 1. But we want to test what happens when
// you have multiple reduces at once.
//
int intermediateReduces = 10;
Path intermediateOuts = new Path(testdir, "intermediateouts");
fs.delete(intermediateOuts, true);
Job checkJob = Job.getInstance(conf);
FileInputFormat.setInputPaths(checkJob, randomOuts);
checkJob.setMapperClass(RandomCheckMapper.class);
FileOutputFormat.setOutputPath(checkJob, intermediateOuts);
checkJob.setOutputKeyClass(IntWritable.class);
checkJob.setOutputValueClass(IntWritable.class);
checkJob.setOutputFormatClass(MapFileOutputFormat.class);
checkJob.setReducerClass(RandomCheckReducer.class);
checkJob.setNumReduceTasks(intermediateReduces);
checkJob.waitForCompletion(true);
printFiles(intermediateOuts, conf);
//
// OK, now we take the output from the last job and
// merge it down to a single file. The map() and reduce()
// functions don't really do anything except reemit tuples.
// But by having a single reduce task here, we end up merging
// all the files.
//
Path finalOuts = new Path(testdir, "finalouts");
fs.delete(finalOuts, true);
Job mergeJob = Job.getInstance(conf);
FileInputFormat.setInputPaths(mergeJob, intermediateOuts);
mergeJob.setInputFormatClass(SequenceFileInputFormat.class);
mergeJob.setMapperClass(MergeMapper.class);
FileOutputFormat.setOutputPath(mergeJob, finalOuts);
mergeJob.setOutputKeyClass(IntWritable.class);
mergeJob.setOutputValueClass(IntWritable.class);
mergeJob.setOutputFormatClass(SequenceFileOutputFormat.class);
mergeJob.setReducerClass(MergeReducer.class);
mergeJob.setNumReduceTasks(1);
mergeJob.waitForCompletion(true);
printFiles(finalOuts, conf);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path recomputedkey = new Path(finalOuts, "part-r-00000");
SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf);
int totalseen = 0;
try {
IntWritable key = new IntWritable();
IntWritable val = new IntWritable();
for (int i = 0; i < range; i++) {
if (dist[i] == 0) {
continue;
}
if (!in.next(key, val)) {
System.err.println("Cannot read entry " + i);
success = false;
break;
} else {
if (!((key.get() == i) && (val.get() == dist[i]))) {
System.err.println("Mismatch! Pos=" + key.get() + ", i=" + i + ", val=" + val.get() + ", dist[i]=" + dist[i]);
success = false;
}
totalseen += val.get();
}
}
if (success) {
if (in.next(key, val)) {
System.err.println("Unnecessary lines in recomputed key!");
success = false;
}
}
} finally {
in.close();
}
int originalTotal = 0;
for (int i = 0; i < dist.length; i++) {
originalTotal += dist[i];
}
System.out.println("Original sum: " + originalTotal);
System.out.println("Recomputed sum: " + totalseen);
//
// Write to "results" whether the test succeeded or not.
//
Path resultFile = new Path(testdir, "results");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile)));
try {
bw.write("Success=" + success + "\n");
System.out.println("Success=" + success);
} finally {
bw.close();
}
assertTrue("testMapRed failed", success);
fs.delete(testdir, true);
We should probably test for more of the file properties.
SATD_ADDED
checkFile(FileSystem, Path, int)
static void checkFile(FileSystem fileSys, Path name, int repl) throws IOException
assertTrue(fileSys.exists(name));
int replication = fileSys.getFileStatus(name).getReplication();
assertEquals("replication for " + name, repl, replication);
// We should probably test for more of the file properties.
VerticaRecord record = new VerticaRecord(null, types, values, date_string);
// TODO: test values as hashmap of column names
// write values into an output buffer
record.write(out);
// copy to an input buffer
in.reset(out.getData(), out.getLength());
// create a new record with new values
List
Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/
SATD_ADDED
testSpaceCommands()
public void testSpaceCommands() throws Exception
final Configuration conf = new HdfsConfiguration();
// set a smaller block size so that we can test with smaller
// diskspace quotas
conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "512");
conf.setBoolean("dfs.support.append", true);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
final FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem);
final DistributedFileSystem dfs = (DistributedFileSystem) fs;
try {
int fileLen = 1024;
short replication = 3;
int fileSpace = fileLen * replication;
// create directory /nqdir0/qdir1/qdir20/nqdir30
assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30")));
// set the quota of /nqdir0/qdir1 to 4 * fileSpace
final Path quotaDir1 = new Path("/nqdir0/qdir1");
dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 4 * fileSpace);
ContentSummary c = dfs.getContentSummary(quotaDir1);
assertEquals(c.getSpaceQuota(), 4 * fileSpace);
// set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace
final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 6 * fileSpace);
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceQuota(), 6 * fileSpace);
// Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace
final Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21");
assertTrue(dfs.mkdirs(quotaDir21));
dfs.setQuota(quotaDir21, FSConstants.QUOTA_DONT_SET, 2 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceQuota(), 2 * fileSpace);
// 5: Create directory /nqdir0/qdir1/qdir21/nqdir32
Path tempPath = new Path(quotaDir21, "nqdir32");
assertTrue(dfs.mkdirs(tempPath));
// create a file under nqdir32/fileDir
DFSTestUtil.createFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication, 0);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), fileSpace);
// Create a larger file /nqdir0/qdir1/qdir21/nqdir33/
boolean hasException = false;
try {
DFSTestUtil.createFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication, 0);
} catch (DSQuotaExceededException e) {
hasException = true;
}
assertTrue(hasException);
// delete nqdir33
assertTrue(dfs.delete(new Path(quotaDir21, "nqdir33"), true));
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), fileSpace);
assertEquals(c.getSpaceQuota(), 2 * fileSpace);
// Verify space before the move:
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 0);
// Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30
Path dstPath = new Path(quotaDir20, "nqdir30");
Path srcPath = new Path(quotaDir21, "nqdir32");
assertTrue(dfs.rename(srcPath, dstPath));
// verify space after the move
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), fileSpace);
// verify space for its parent
c = dfs.getContentSummary(quotaDir1);
assertEquals(c.getSpaceConsumed(), fileSpace);
// verify space for source for the move
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
final Path file2 = new Path(dstPath, "fileDir/file2");
int file2Len = 2 * fileLen;
// create a larger file under /nqdir0/qdir1/qdir20/nqdir30
DFSTestUtil.createFile(dfs, file2, file2Len, replication, 0);
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
// Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/
hasException = false;
try {
assertFalse(dfs.rename(dstPath, srcPath));
} catch (DSQuotaExceededException e) {
hasException = true;
}
assertTrue(hasException);
// make sure no intermediate directories left by failed rename
assertFalse(dfs.exists(srcPath));
// directory should exist
assertTrue(dfs.exists(dstPath));
// verify space after the failed move
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
// Test Append :
// verify space quota
c = dfs.getContentSummary(quotaDir1);
assertEquals(c.getSpaceQuota(), 4 * fileSpace);
// verify space before append;
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
OutputStream out = dfs.append(file2);
// appending 1 fileLen should succeed
out.write(new byte[fileLen]);
out.close();
// after append
file2Len += fileLen;
// verify space after append;
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 4 * fileSpace);
// now increase the quota for quotaDir1
dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 5 * fileSpace);
// Now, appending more than 1 fileLen should result in an error
out = dfs.append(file2);
hasException = false;
try {
out.write(new byte[fileLen + 1024]);
out.flush();
out.close();
} catch (DSQuotaExceededException e) {
hasException = true;
IOUtils.closeStream(out);
}
assertTrue(hasException);
// after partial append
file2Len += fileLen;
// verify space after partial append
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace);
// Test set replication :
// first reduce the replication
dfs.setReplication(file2, (short) (replication - 1));
// verify that space is reduced by file2Len
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now try to increase the replication and and expect an error.
hasException = false;
try {
dfs.setReplication(file2, (short) (replication + 1));
} catch (DSQuotaExceededException e) {
hasException = true;
}
assertTrue(hasException);
// verify space consumed remains unchanged.
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now increase the quota for quotaDir1 and quotaDir20
dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 10 * fileSpace);
dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 10 * fileSpace);
// then increasing replication should be ok.
dfs.setReplication(file2, (short) (replication + 1));
// verify increase in space
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace + file2Len);
} finally {
cluster.shutdown();
}
final Configuration conf = new HdfsConfiguration();
// set a smaller block size so that we can test with smaller
// diskspace quotas
conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "512");
conf.setBoolean("dfs.support.append", true);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
final FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: " + fs.getUri(), fs instanceof DistributedFileSystem);
final DistributedFileSystem dfs = (DistributedFileSystem) fs;
try {
int fileLen = 1024;
short replication = 3;
int fileSpace = fileLen * replication;
// create directory /nqdir0/qdir1/qdir20/nqdir30
assertTrue(dfs.mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30")));
// set the quota of /nqdir0/qdir1 to 4 * fileSpace
final Path quotaDir1 = new Path("/nqdir0/qdir1");
dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 4 * fileSpace);
ContentSummary c = dfs.getContentSummary(quotaDir1);
assertEquals(c.getSpaceQuota(), 4 * fileSpace);
// set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace
final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 6 * fileSpace);
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceQuota(), 6 * fileSpace);
// Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace
final Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21");
assertTrue(dfs.mkdirs(quotaDir21));
dfs.setQuota(quotaDir21, FSConstants.QUOTA_DONT_SET, 2 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceQuota(), 2 * fileSpace);
// 5: Create directory /nqdir0/qdir1/qdir21/nqdir32
Path tempPath = new Path(quotaDir21, "nqdir32");
assertTrue(dfs.mkdirs(tempPath));
// create a file under nqdir32/fileDir
DFSTestUtil.createFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication, 0);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), fileSpace);
// Create a larger file /nqdir0/qdir1/qdir21/nqdir33/
boolean hasException = false;
try {
DFSTestUtil.createFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication, 0);
} catch (DSQuotaExceededException e) {
hasException = true;
}
assertTrue(hasException);
// delete nqdir33
assertTrue(dfs.delete(new Path(quotaDir21, "nqdir33"), true));
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), fileSpace);
assertEquals(c.getSpaceQuota(), 2 * fileSpace);
// Verify space before the move:
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 0);
// Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30
Path dstPath = new Path(quotaDir20, "nqdir30");
Path srcPath = new Path(quotaDir21, "nqdir32");
assertTrue(dfs.rename(srcPath, dstPath));
// verify space after the move
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), fileSpace);
// verify space for its parent
c = dfs.getContentSummary(quotaDir1);
assertEquals(c.getSpaceConsumed(), fileSpace);
// verify space for source for the move
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
final Path file2 = new Path(dstPath, "fileDir/file2");
int file2Len = 2 * fileLen;
// create a larger file under /nqdir0/qdir1/qdir20/nqdir30
DFSTestUtil.createFile(dfs, file2, file2Len, replication, 0);
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
// Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/
hasException = false;
try {
assertFalse(dfs.rename(dstPath, srcPath));
} catch (DSQuotaExceededException e) {
hasException = true;
}
assertTrue(hasException);
// make sure no intermediate directories left by failed rename
assertFalse(dfs.exists(srcPath));
// directory should exist
assertTrue(dfs.exists(dstPath));
// verify space after the failed move
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
// Test Append :
// verify space quota
c = dfs.getContentSummary(quotaDir1);
assertEquals(c.getSpaceQuota(), 4 * fileSpace);
// verify space before append;
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
OutputStream out = dfs.append(file2);
// appending 1 fileLen should succeed
out.write(new byte[fileLen]);
out.close();
// after append
file2Len += fileLen;
// verify space after append;
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 4 * fileSpace);
// now increase the quota for quotaDir1
dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 5 * fileSpace);
// Now, appending more than 1 fileLen should result in an error
out = dfs.append(file2);
hasException = false;
try {
out.write(new byte[fileLen + 1024]);
out.flush();
out.close();
} catch (DSQuotaExceededException e) {
hasException = true;
IOUtils.closeStream(out);
}
assertTrue(hasException);
// after partial append
file2Len += fileLen;
// verify space after partial append
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace);
// Test set replication :
// first reduce the replication
dfs.setReplication(file2, (short) (replication - 1));
// verify that space is reduced by file2Len
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now try to increase the replication and and expect an error.
hasException = false;
try {
dfs.setReplication(file2, (short) (replication + 1));
} catch (DSQuotaExceededException e) {
hasException = true;
}
assertTrue(hasException);
// verify space consumed remains unchanged.
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now increase the quota for quotaDir1 and quotaDir20
dfs.setQuota(quotaDir1, FSConstants.QUOTA_DONT_SET, 10 * fileSpace);
dfs.setQuota(quotaDir20, FSConstants.QUOTA_DONT_SET, 10 * fileSpace);
// then increasing replication should be ok.
dfs.setReplication(file2, (short) (replication + 1));
// verify increase in space
c = dfs.getContentSummary(dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace + file2Len);
} finally {
cluster.shutdown();
}
As we already know current Capacity percent of this queue
make children distribute unconfigured Capacity.
SATD_ADDED
distributeUnConfiguredCapacity()
void distributeUnConfiguredCapacity()
List unConfiguredQueues = new ArrayList();
float totalCapacity = 0;
for (AbstractQueue q : children) {
if (q.qsc.getCapacityPercent() == -1) {
// Add into unConfigured queue.
unConfiguredQueues.add(q);
} else {
// If capacity is set , then add that to totalCapacity.
LOG.info(" the capacity percent of the queue " + q.getName() + " is " + "" + q.qsc.getCapacityPercent());
totalCapacity += q.qsc.getCapacityPercent();
// As we already know current Capacity percent of this queue
// make children distribute unconfigured Capacity.
q.distributeUnConfiguredCapacity();
}
}
if (!unConfiguredQueues.isEmpty()) {
LOG.info("Total capacity to be distributed among the others are " + "" + (100 - totalCapacity));
// We have list of queues at this level which are unconfigured.
// 100 - totalCapacity is the capacity remaining.
// Divide it equally among all the un configured queues.
float capacityShare = (100 - totalCapacity) / unConfiguredQueues.size();
// We dont have to check for 100 - totalCapacity being -ve , as
// we already do it while loading.
for (AbstractQueue q : unConfiguredQueues) {
if (q.qsc.getMaxCapacityPercent() > 0) {
if (q.qsc.getMaxCapacityPercent() < capacityShare) {
throw new IllegalStateException(" Capacity share (" + capacityShare + ")for unconfigured queue " + q.getName() + " is greater than its maximum-capacity percentage " + q.qsc.getMaxCapacityPercent());
}
}
q.qsc.setCapacityPercent(capacityShare);
LOG.info("Capacity share for un configured queue " + q.getName() + "" + " is " + capacityShare);
// we have q's capacity now.
// make children also distribute it among themselves.
q.distributeUnConfiguredCapacity();
}
}
// Set the job up.
final Configuration myConf = mr.createJobConf();
myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague);
// Submit the job as user1
Job job = submitJobAsUser(myConf, jobSubmitter);
final JobID jobId = job.getJobID();
// Try operations as an unauthorized user.
verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague);
// Try operations as an authorized user, who is part of view-job-acl.
verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague);
// Try operations as an authorized user, who is a queue administrator.
verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin);
// Clean up the job
job.killJob();
configure task jvm options if enabled
this knob can be turned off if there is a mismatch between the
target (simulation) cluster and the original cluster. Such a
mismatch can result in job failures (due to memory issues) on the
target (simulated) cluster.
TODO If configured, scale the original task's JVM (heap related)
options to suit the target (simulation) cluster
SATD_ADDED
initialValue()
protected Formatter initialValue()
final StringBuilder sb = new StringBuilder(JOB_NAME_PREFIX.length() + 6);
sb.append(JOB_NAME_PREFIX);
return new Formatter(sb);
// Set the job up.
final Configuration myConf = mr.createJobConf();
myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague);
// Submit the job as user1
Job job = submitJobAsUser(myConf, jobSubmitter);
final JobID jobId = job.getJobID();
// Try operations as an unauthorized user.
verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague);
// Try operations as an authorized user, who is part of view-job-acl.
verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague);
// Try operations as an authorized user, who is a queue administrator.
verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin);
// Clean up the job
job.killJob();
public void testTotalHeapUsageEmulatorPlugin() throws Exception
Configuration conf = new Configuration();
// set the dummy resource calculator for testing
ResourceCalculatorPlugin monitor = new DummyResourceCalculatorPlugin();
// 1GB
long maxHeapUsage = 1024 * TotalHeapUsageEmulatorPlugin.ONE_MB;
conf.setLong(DummyResourceCalculatorPlugin.MAXPMEM_TESTING_PROPERTY, maxHeapUsage);
monitor.setConf(conf);
// no buffer to be reserved
conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
// only 1 call to be made per cycle
conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
// 200mb
long targetHeapUsageInMB = 200;
// fake progress indicator
FakeProgressive fakeProgress = new FakeProgressive();
// fake heap usage generator
FakeHeapUsageEmulatorCore fakeCore = new FakeHeapUsageEmulatorCore();
// a heap usage emulator with fake core
FakeHeapUsageEmulatorPlugin heapPlugin = new FakeHeapUsageEmulatorPlugin(fakeCore);
// test with invalid or missing resource usage value
ResourceUsageMetrics invalidUsage = TestResourceUsageEmulators.createMetrics(0);
heapPlugin.initialize(conf, invalidUsage, null, null);
// test if disabled heap emulation plugin's emulate() call is a no-operation
// this will test if the emulation plugin is disabled or not
int numCallsPre = fakeCore.getNumCalls();
long heapUsagePre = fakeCore.getHeapUsageInMB();
heapPlugin.emulate();
int numCallsPost = fakeCore.getNumCalls();
long heapUsagePost = fakeCore.getHeapUsageInMB();
// test if no calls are made heap usage emulator core
assertEquals("Disabled heap usage emulation plugin works!", numCallsPre, numCallsPost);
// test if no calls are made heap usage emulator core
assertEquals("Disabled heap usage emulation plugin works!", heapUsagePre, heapUsagePost);
// test with wrong/invalid configuration
Boolean failed = null;
invalidUsage = TestResourceUsageEmulators.createMetrics(maxHeapUsage + TotalHeapUsageEmulatorPlugin.ONE_MB);
try {
heapPlugin.initialize(conf, invalidUsage, monitor, null);
failed = false;
} catch (Exception e) {
failed = true;
}
assertNotNull("Fail case failure!", failed);
assertTrue("Expected failure!", failed);
// test with valid resource usage value
ResourceUsageMetrics metrics = TestResourceUsageEmulators.createMetrics(targetHeapUsageInMB * TotalHeapUsageEmulatorPlugin.ONE_MB);
// test with default emulation interval
// in every interval, the emulator will add 100% of the expected usage
// (since gridmix.emulators.resource-usage.heap.load-ratio=1)
// so at 10%, emulator will add 10% (difference), at 20% it will add 10% ...
// So to emulate 200MB, it will add
// 20mb + 20mb + 20mb + 20mb + .. = 200mb
testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10);
// test with custom value for emulation interval of 20%
conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.2F);
// 40mb + 40mb + 40mb + 40mb + 40mb = 200mb
testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 5);
// test with custom value of free heap ratio and load ratio = 1
conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.5F);
// 40mb + 0mb + 80mb + 0mb + 0mb = 120mb
testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 120, 2);
// test with custom value of heap load ratio and min free heap ratio = 0
conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F);
conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
// 20mb (call#1) + 20mb (call#1) + 20mb (call#2) + 20mb (call#2) +.. = 200mb
testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10);
// test with custom value of free heap ratio = 0.3 and load ratio = 0.5
conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.25F);
conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F);
// 20mb (call#1) + 20mb (call#1) + 30mb (call#2) + 0mb (call#2)
// + 30mb (call#3) + 0mb (call#3) + 35mb (call#4) + 0mb (call#4)
// + 37mb (call#5) + 0mb (call#5) = 162mb
testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 162, 6);
// test if emulation interval boundary is respected
// initialize
fakeProgress = new FakeProgressive();
conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.25F);
heapPlugin.initialize(conf, metrics, monitor, fakeProgress);
fakeCore.resetFake();
// take a snapshot after the initialization
long initHeapUsage = fakeCore.getHeapUsageInMB();
long initNumCallsUsage = fakeCore.getNumCalls();
// test with 0 progress
testEmulationBoundary(0F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 0 progress]");
// test with 24% progress
testEmulationBoundary(0.24F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 24% progress]");
// test with 25% progress
testEmulationBoundary(0.25F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB / 4, 1, "[op, 25% progress]");
// test with 80% progress
testEmulationBoundary(0.80F, fakeCore, fakeProgress, heapPlugin, (targetHeapUsageInMB * 4) / 5, 2, "[op, 80% progress]");
// now test if the final call with 100% progress ramps up the heap usage
testEmulationBoundary(1F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB, 3, "[op, 100% progress]");
Hack for local FS that does not have the concept of a 'mounting point'
SATD_ADDED
launchWordCount(JobConf, String, int, int)
private String launchWordCount(JobConf conf, String input, int numMaps, int numReduces) throws IOException
Path inDir = new Path("testing/wc/input");
Path outDir = new Path("testing/wc/output");
// Hack for local FS that does not have the concept of a 'mounting point'
if (isLocalFS()) {
String localPathRoot = System.getProperty("test.build.data", "/tmp").toString().replace(' ', '+');
;
inDir = new Path(localPathRoot, inDir);
outDir = new Path(localPathRoot, outDir);
}
FileSystem fs = FileSystem.get(conf);
fs.delete(outDir, true);
if (!fs.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
{
DataOutputStream file = fs.create(new Path(inDir, "part-0"));
file.writeBytes(input);
file.close();
}
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
// the keys are words (strings)
conf.setOutputKeyClass(Text.class);
// the values are counts (ints)
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(WordCount.MapClass.class);
conf.setCombinerClass(WordCount.Reduce.class);
conf.setReducerClass(WordCount.Reduce.class);
FileInputFormat.setInputPaths(conf, inDir);
FileOutputFormat.setOutputPath(conf, outDir);
conf.setNumMapTasks(numMaps);
conf.setNumReduceTasks(numReduces);
JobClient.runJob(conf);
return MapReduceTestUtil.readOutput(outDir, conf);
Hack for local FS that does not have the concept of a 'mounting point'
SATD_ADDED
testMR()
public void testMR() throws Exception
System.out.println(launchWordCount(this.createJobConf(), "a b c d e f g h", 1, 1));
Thread.sleep(2000);
assertEquals(2, NotificationServlet.counter);
Path inDir = new Path("notificationjob/input");
Path outDir = new Path("notificationjob/output");
// Hack for local FS that does not have the concept of a 'mounting point'
if (isLocalFS()) {
String localPathRoot = System.getProperty("test.build.data", "/tmp").toString().replace(' ', '+');
;
inDir = new Path(localPathRoot, inDir);
outDir = new Path(localPathRoot, outDir);
}
// run a job with KILLED status
System.out.println(UtilsForTests.runJobKill(this.createJobConf(), inDir, outDir).getID());
Thread.sleep(2000);
assertEquals(4, NotificationServlet.counter);
// run a job with FAILED status
System.out.println(UtilsForTests.runJobFail(this.createJobConf(), inDir, outDir).getID());
Thread.sleep(2000);
assertEquals(6, NotificationServlet.counter);
Poll the Namenode (once every 5 minutes) to find the size of the
pending edit log.
5 minutes
SATD_ADDED
doWork()
public void doWork()
//
// Poll the Namenode (once every 5 minutes) to find the size of the
// pending edit log.
//
// 5 minutes
long period = 5 * 60;
if (checkpointPeriod < period) {
period = checkpointPeriod;
}
while (shouldRun) {
try {
Thread.sleep(1000 * period);
} catch (InterruptedException ie) {
// do nothing
}
if (!shouldRun) {
break;
}
try {
// We may have lost our ticket since last checkpoint, log in again, just in case
if (UserGroupInformation.isSecurityEnabled())
UserGroupInformation.getCurrentUser().reloginFromKeytab();
long now = System.currentTimeMillis();
long size = namenode.getEditLogSize();
if (size >= checkpointSize || now >= lastCheckpointTime + 1000 * checkpointPeriod) {
doCheckpoint();
lastCheckpointTime = now;
}
} catch (IOException e) {
LOG.error("Exception in doCheckpoint: ");
LOG.error(StringUtils.stringifyException(e));
e.printStackTrace();
checkpointImage.getStorage().imageDigest = null;
} catch (Throwable e) {
LOG.error("Throwable Exception in doCheckpoint: ");
LOG.error(StringUtils.stringifyException(e));
e.printStackTrace();
Runtime.getRuntime().exit(-1);
}
}
void syncBlock(RecoveringBlock rBlock, List syncList) throws IOException
ExtendedBlock block = rBlock.getBlock();
DatanodeProtocol nn = getBPNamenode(block.getBlockPoolId());
long recoveryId = rBlock.getNewGenerationStamp();
if (LOG.isDebugEnabled()) {
LOG.debug("block=" + block + ", (length=" + block.getNumBytes() + "), syncList=" + syncList);
}
// syncList.isEmpty() means that all data-nodes do not have the block
// or their replicas have 0 length.
// The block can be deleted.
if (syncList.isEmpty()) {
nn.commitBlockSynchronization(block, recoveryId, 0, true, true, DatanodeID.EMPTY_ARRAY);
return;
}
// Calculate the best available replica state.
ReplicaState bestState = ReplicaState.RWR;
long finalizedLength = -1;
for (BlockRecord r : syncList) {
assert r.rInfo.getNumBytes() > 0 : "zero length replica";
ReplicaState rState = r.rInfo.getOriginalReplicaState();
if (rState.getValue() < bestState.getValue())
bestState = rState;
if (rState == ReplicaState.FINALIZED) {
if (finalizedLength > 0 && finalizedLength != r.rInfo.getNumBytes())
throw new IOException("Inconsistent size of finalized replicas. " + "Replica " + r.rInfo + " expected size: " + finalizedLength);
finalizedLength = r.rInfo.getNumBytes();
}
}
// Calculate list of nodes that will participate in the recovery
// and the new block size
List participatingList = new ArrayList();
final ExtendedBlock newBlock = new ExtendedBlock(block.getBlockPoolId(), block.getBlockId(), -1, recoveryId);
switch(bestState) {
case FINALIZED:
assert finalizedLength > 0 : "finalizedLength is not positive";
for (BlockRecord r : syncList) {
ReplicaState rState = r.rInfo.getOriginalReplicaState();
if (rState == ReplicaState.FINALIZED || rState == ReplicaState.RBW && r.rInfo.getNumBytes() == finalizedLength)
participatingList.add(r);
}
newBlock.setNumBytes(finalizedLength);
break;
case RBW:
case RWR:
long minLength = Long.MAX_VALUE;
for (BlockRecord r : syncList) {
ReplicaState rState = r.rInfo.getOriginalReplicaState();
if (rState == bestState) {
minLength = Math.min(minLength, r.rInfo.getNumBytes());
participatingList.add(r);
}
}
newBlock.setNumBytes(minLength);
break;
case RUR:
case TEMPORARY:
assert false : "bad replica state: " + bestState;
}
List failedList = new ArrayList();
List successList = new ArrayList();
for (BlockRecord r : participatingList) {
try {
ExtendedBlock reply = r.datanode.updateReplicaUnderRecovery(new ExtendedBlock(newBlock.getBlockPoolId(), r.rInfo), recoveryId, newBlock.getNumBytes());
assert reply.equals(newBlock) && reply.getNumBytes() == newBlock.getNumBytes() : "Updated replica must be the same as the new block.";
successList.add(r.id);
} catch (IOException e) {
InterDatanodeProtocol.LOG.warn("Failed to updateBlock (newblock=" + newBlock + ", datanode=" + r.id + ")", e);
failedList.add(r.id);
}
}
// If any of the data-nodes failed, the recovery fails, because
// we never know the actual state of the replica on failed data-nodes.
// The recovery should be started over.
if (!failedList.isEmpty()) {
StringBuilder b = new StringBuilder();
for (DatanodeID id : failedList) {
b.append("\n " + id);
}
throw new IOException("Cannot recover " + block + ", the following " + failedList.size() + " data-nodes failed {" + b + "\n}");
}
// Notify the name-node about successfully recovered replicas.
DatanodeID[] nlist = successList.toArray(new DatanodeID[successList.size()]);
nn.commitBlockSynchronization(block, newBlock.getGenerationStamp(), newBlock.getNumBytes(), true, false, nlist);
Return
\"DS-randInt-ipaddr-currentTimeMillis\"
It is considered extermely rare for all these numbers to match
on a different machine accidentally for the following
a) SecureRandom(INT_MAX) is pretty much random (1 in 2 billion), and
b) Good chance ip address would be different, and
c) Even on the same machine, Datanode is designed to use different ports.
d) Good chance that these are started at different times.
For a confict to occur all the 4 above have to match!.
The format of this string can be changed anytime in future without
affecting its functionality.
SATD_ADDED
createSocketAddr(String)
public static InetSocketAddress createSocketAddr(String target) throws IOException
private TrieNode buildTrieRec(BinaryComparable[] splits, int lower, int upper, byte[] prefix, int maxDepth, CarriedTrieNodeRef ref)
final int depth = prefix.length;
// We generate leaves for a single split point as well as for
// no split points.
if (depth >= maxDepth || lower >= upper - 1) {
// If we have two consecutive requests for an unsplit trie node, we
// can deliver the same one the second time.
if (lower == upper && ref.content != null) {
return ref.content;
}
TrieNode result = LeafTrieNodeFactory(depth, splits, lower, upper);
ref.content = lower == upper ? result : null;
return result;
}
InnerTrieNode result = new InnerTrieNode(depth);
byte[] trial = Arrays.copyOf(prefix, prefix.length + 1);
// append an extra byte on to the prefix
int currentBound = lower;
for (int ch = 0; ch < 0xFF; ++ch) {
trial[depth] = (byte) (ch + 1);
lower = currentBound;
while (currentBound < upper) {
if (splits[currentBound].compareTo(trial, 0, trial.length) >= 0) {
break;
}
currentBound += 1;
}
trial[depth] = (byte) ch;
result.child[0xFF & ch] = buildTrieRec(splits, lower, currentBound, trial, maxDepth, ref);
}
// pick up the rest
trial[depth] = (byte) 0xFF;
result.child[0xFF] = buildTrieRec(splits, lower, currentBound, trial, maxDepth, ref);
return result;
REMIND - mjc - eventually we should have a timeout system
in place to clean up block files left by abandoned clients.
We should have some timer in place, so that if a blockfile
is created but non-valid, and has been idle for >48 hours,
we can GC it safely.
SATD_ADDED
addBlock(Block, File)
public File addBlock(Block b, File src) throws IOException
Files that were being written when the datanode was last shutdown
are now moved back to the data directory. It is possible that
in the future, we might want to do some sort of datanode-local
recovery for these blocks. For example, crc validation.
SATD_ADDED
addBlock(Block, File)
public File addBlock(Block b, File src) throws IOException
for security authorization
server principal for this call
should be JT's one.
SATD_ADDED
refreshUserToGroupsMappings()
private int refreshUserToGroupsMappings() throws IOException
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be JT's one.
JobConf jConf = new JobConf(conf);
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, ""));
// Create the client
RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class));
// Refresh the user-to-groups mappings
refreshProtocol.refreshUserToGroupsMappings();
return 0;
for security authorization
server principal for this call
should be JT's one.
SATD_ADDED
refreshSuperUserGroupsConfiguration()
public int refreshSuperUserGroupsConfiguration() throws IOException
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be JT's one.
JobConf jConf = new JobConf(conf);
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, ""));
// Create the client
RefreshUserMappingsProtocol refreshProtocol = (RefreshUserMappingsProtocol) RPC.getProxy(RefreshUserMappingsProtocol.class, RefreshUserMappingsProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshUserMappingsProtocol.class));
// Refresh the user-to-groups mappings
refreshProtocol.refreshSuperUserGroupsConfiguration();
return 0;
private int refreshAuthorizationPolicy() throws IOException
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be JT's one.
JobConf jConf = new JobConf(conf);
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, ""));
// Create the client
RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class));
// Refresh the authorization policy in-effect
refreshProtocol.refreshServiceAcl();
return 0;
for security authorization
server principal for this call
should be JT's one.
SATD_ADDED
refreshAuthorizationPolicy()
private int refreshAuthorizationPolicy() throws IOException
// Get the current configuration
Configuration conf = getConf();
// for security authorization
// server principal for this call
// should be JT's one.
JobConf jConf = new JobConf(conf);
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, jConf.get(JobTracker.JT_USER_NAME, ""));
// Create the client
RefreshAuthorizationPolicyProtocol refreshProtocol = (RefreshAuthorizationPolicyProtocol) RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, RefreshAuthorizationPolicyProtocol.versionID, JobTracker.getAddress(conf), getUGI(conf), conf, NetUtils.getSocketFactory(conf, RefreshAuthorizationPolicyProtocol.class));
// Refresh the authorization policy in-effect
refreshProtocol.refreshServiceAcl();
return 0;
if there are more numSplits than params we're going to introduce some
limit and offsets
TODO: write code to generate the start/end pairs for each group
SATD_ADDED
getSplits(JobContext)
public static List getSplits(JobContext context) throws IOException
Configuration conf = context.getConfiguration();
int numSplits = conf.getInt("mapreduce.job.maps", 1);
LOG.debug("creating splits up to " + numSplits);
List splits = new ArrayList();
int i = 0;
long start = 0;
long end = 0;
boolean limitOffset = true;
// This is the fancy part of mapping inputs...here's how we figure out
// splits
// get the params query or the params
VerticaConfiguration config = new VerticaConfiguration(conf);
String inputQuery = config.getInputQuery();
if (inputQuery == null)
throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP);
String paramsQuery = config.getParamsQuery();
Collection> params = config.getInputParameters();
// TODO: limit needs order by unique key
// TODO: what if there are more parameters than numsplits?
// prep a count(*) wrapper query and then populate the bind params for each
String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count";
if (paramsQuery != null) {
LOG.debug("creating splits using paramsQuery :" + paramsQuery);
Connection conn = null;
Statement stmt = null;
try {
conn = config.getConnection(false);
stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(paramsQuery);
ResultSetMetaData rsmd = rs.getMetaData();
while (rs.next()) {
limitOffset = false;
List
TODO: limit needs order by unique key
TODO: what if there are more parameters than numsplits?
prep a count(*) wrapper query and then populate the bind params for each
SATD_ADDED
getSplits(JobContext)
public static List getSplits(JobContext context) throws IOException
Configuration conf = context.getConfiguration();
int numSplits = conf.getInt("mapreduce.job.maps", 1);
LOG.debug("creating splits up to " + numSplits);
List splits = new ArrayList();
int i = 0;
long start = 0;
long end = 0;
boolean limitOffset = true;
// This is the fancy part of mapping inputs...here's how we figure out
// splits
// get the params query or the params
VerticaConfiguration config = new VerticaConfiguration(conf);
String inputQuery = config.getInputQuery();
if (inputQuery == null)
throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP);
String paramsQuery = config.getParamsQuery();
Collection> params = config.getInputParameters();
// TODO: limit needs order by unique key
// TODO: what if there are more parameters than numsplits?
// prep a count(*) wrapper query and then populate the bind params for each
String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count";
if (paramsQuery != null) {
LOG.debug("creating splits using paramsQuery :" + paramsQuery);
Connection conn = null;
Statement stmt = null;
try {
conn = config.getConnection(false);
stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(paramsQuery);
ResultSetMetaData rsmd = rs.getMetaData();
while (rs.next()) {
limitOffset = false;
List
This is the fancy part of mapping inputs...here's how we figure out
splits
get the params query or the params
SATD_ADDED
getSplits(JobContext)
public static List getSplits(JobContext context) throws IOException
Configuration conf = context.getConfiguration();
int numSplits = conf.getInt("mapreduce.job.maps", 1);
LOG.debug("creating splits up to " + numSplits);
List splits = new ArrayList();
int i = 0;
long start = 0;
long end = 0;
boolean limitOffset = true;
// This is the fancy part of mapping inputs...here's how we figure out
// splits
// get the params query or the params
VerticaConfiguration config = new VerticaConfiguration(conf);
String inputQuery = config.getInputQuery();
if (inputQuery == null)
throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP);
String paramsQuery = config.getParamsQuery();
Collection> params = config.getInputParameters();
// TODO: limit needs order by unique key
// TODO: what if there are more parameters than numsplits?
// prep a count(*) wrapper query and then populate the bind params for each
String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count";
if (paramsQuery != null) {
LOG.debug("creating splits using paramsQuery :" + paramsQuery);
Connection conn = null;
Statement stmt = null;
try {
conn = config.getConnection(false);
stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(paramsQuery);
ResultSetMetaData rsmd = rs.getMetaData();
while (rs.next()) {
limitOffset = false;
List
TODO: catch when params required but missing
TODO: better error message when count query is bad
SATD_ADDED
getSplits(JobContext)
public static List getSplits(JobContext context) throws IOException
Configuration conf = context.getConfiguration();
int numSplits = conf.getInt("mapreduce.job.maps", 1);
LOG.debug("creating splits up to " + numSplits);
List splits = new ArrayList();
int i = 0;
long start = 0;
long end = 0;
boolean limitOffset = true;
// This is the fancy part of mapping inputs...here's how we figure out
// splits
// get the params query or the params
VerticaConfiguration config = new VerticaConfiguration(conf);
String inputQuery = config.getInputQuery();
if (inputQuery == null)
throw new IOException("Vertica input requires query defined by " + VerticaConfiguration.QUERY_PROP);
String paramsQuery = config.getParamsQuery();
Collection> params = config.getInputParameters();
// TODO: limit needs order by unique key
// TODO: what if there are more parameters than numsplits?
// prep a count(*) wrapper query and then populate the bind params for each
String countQuery = "SELECT COUNT(*) FROM (\n" + inputQuery + "\n) count";
if (paramsQuery != null) {
LOG.debug("creating splits using paramsQuery :" + paramsQuery);
Connection conn = null;
Statement stmt = null;
try {
conn = config.getConnection(false);
stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(paramsQuery);
ResultSetMetaData rsmd = rs.getMetaData();
while (rs.next()) {
limitOffset = false;
List
spawn the JVM in a new thread. Note that there will be very little
extra overhead of launching the new thread for a new JVM since
most of the cost is involved in launching the process. Moreover,
since we are going to be using the JVM for running many tasks,
the thread launch cost becomes trivial when amortized over all
tasks. Doing it this way also keeps code simple.
long parentNamespace = counts.nsCount;
long parentDiskspace = counts.dsCount;
// for self. should not call node.spaceConsumedInTree()
counts.nsCount = 1L;
counts.dsCount = 0L;
/* We don't need nodesInPath if we could use 'parent' field in
* INode. using 'parent' is not currently recommended. */
nodesInPath.add(dir);
for (INode child : dir.getChildren()) {
if (child.isDirectory()) {
updateCountForINodeWithQuota((INodeDirectory) child, counts, nodesInPath);
} else if (child.isLink()) {
counts.nsCount += 1;
} else {
// reduce recursive calls
counts.nsCount += 1;
counts.dsCount += ((INodeFile) child).diskspaceConsumed();
}
}
if (dir.isQuotaSet()) {
((INodeDirectoryWithQuota) dir).setSpaceConsumed(counts.nsCount, counts.dsCount);
// check if quota is violated for some reason.
if ((dir.getNsQuota() >= 0 && counts.nsCount > dir.getNsQuota()) || (dir.getDsQuota() >= 0 && counts.dsCount > dir.getDsQuota())) {
// can only happen because of a software bug. the bug should be fixed.
StringBuilder path = new StringBuilder(512);
for (INode n : nodesInPath) {
path.append('/');
path.append(n.getLocalName());
}
NameNode.LOG.warn("Quota violation in image for " + path + " (Namespace quota : " + dir.getNsQuota() + " consumed : " + counts.nsCount + ")" + " (Diskspace quota : " + dir.getDsQuota() + " consumed : " + counts.dsCount + ").");
}
}
// pop
nodesInPath.remove(nodesInPath.size() - 1);
counts.nsCount += parentNamespace;
counts.dsCount += parentDiskspace;
public void unprotectedConcat(String target, String[] srcs, long timestamp) throws UnresolvedLinkException
assert hasWriteLock();
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSNamesystem.concat to " + target);
}
// do the move
INode[] trgINodes = getExistingPathINodes(target);
INodeFile trgInode = (INodeFile) trgINodes[trgINodes.length - 1];
INodeDirectory trgParent = (INodeDirectory) trgINodes[trgINodes.length - 2];
INodeFile[] allSrcInodes = new INodeFile[srcs.length];
int i = 0;
int totalBlocks = 0;
for (String src : srcs) {
INodeFile srcInode = getFileINode(src);
allSrcInodes[i++] = srcInode;
totalBlocks += srcInode.blocks.length;
}
// copy the blocks
trgInode.appendBlocks(allSrcInodes, totalBlocks);
// since we are in the same dir - we can use same parent to remove files
int count = 0;
for (INodeFile nodeToRemove : allSrcInodes) {
if (nodeToRemove == null)
continue;
nodeToRemove.blocks = null;
trgParent.removeChild(nodeToRemove);
count++;
}
trgInode.setModificationTimeForce(timestamp);
trgParent.setModificationTime(timestamp);
// update quota on the parent directory ('count' files removed, 0 space)
unprotectedUpdateCount(trgINodes, trgINodes.length - 1, -count, 0);
Check the permissions of the task-controller binary by running it plainly.
If permissions are correct, it returns an error code 1, else it returns
24 or something else if some other bugs are also present.
SATD_ADDED
setup()
public void setup() throws IOException
super.setup();
// Check the permissions of the task-controller binary by running it plainly.
// If permissions are correct, it returns an error code 1, else it returns
// 24 or something else if some other bugs are also present.
String[] taskControllerCmd = new String[] { getTaskControllerExecutablePath() };
ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd);
try {
shExec.execute();
} catch (ExitCodeException e) {
int exitCode = shExec.getExitCode();
if (exitCode != 1) {
LOG.warn("Exit code from checking binary permissions is : " + exitCode);
logOutput(shExec.getOutput());
throw new IOException("Task controller setup failed because of invalid" + "permissions/ownership with exit code " + exitCode, e);
}
}
Split the special file that contains the list of distributed cache file
paths and their file sizes such that each split corresponds to
approximately same amount of distributed cache data to be generated.
Consider numTaskTrackers * numMapSlotsPerTracker as the number of maps
for this job, if there is lot of data to be generated.
SATD_ADDED
getSplits(JobContext)
public List getSplits(JobContext jobCtxt) throws IOException
final JobConf jobConf = new JobConf(jobCtxt.getConfiguration());
final JobClient client = new JobClient(jobConf);
ClusterStatus stat = client.getClusterStatus(true);
int numTrackers = stat.getTaskTrackers();
final int fileCount = jobConf.getInt(GRIDMIX_DISTCACHE_FILE_COUNT, -1);
// Total size of distributed cache files to be generated
final long totalSize = jobConf.getLong(GRIDMIX_DISTCACHE_BYTE_COUNT, -1);
// Get the path of the special file
String distCacheFileList = jobConf.get(GRIDMIX_DISTCACHE_FILE_LIST);
if (fileCount < 0 || totalSize < 0 || distCacheFileList == null) {
throw new RuntimeException("Invalid metadata: #files (" + fileCount + "), total_size (" + totalSize + "), filelisturi (" + distCacheFileList + ")");
}
Path sequenceFile = new Path(distCacheFileList);
FileSystem fs = sequenceFile.getFileSystem(jobConf);
FileStatus srcst = fs.getFileStatus(sequenceFile);
// Consider the number of TTs * mapSlotsPerTracker as number of mappers.
int numMapSlotsPerTracker = jobConf.getInt(TTConfig.TT_MAP_SLOTS, 2);
int numSplits = numTrackers * numMapSlotsPerTracker;
List splits = new ArrayList(numSplits);
LongWritable key = new LongWritable();
BytesWritable value = new BytesWritable();
// Average size of data to be generated by each map task
final long targetSize = Math.max(totalSize / numSplits, DistributedCacheEmulator.AVG_BYTES_PER_MAP);
long splitStartPosition = 0L;
long splitEndPosition = 0L;
long acc = 0L;
long bytesRemaining = srcst.getLen();
SequenceFile.Reader reader = null;
try {
reader = new SequenceFile.Reader(fs, sequenceFile, jobConf);
while (reader.next(key, value)) {
// If adding this file would put this split past the target size,
// cut the last split and put this file in the next split.
if (acc + key.get() > targetSize && acc != 0) {
long splitSize = splitEndPosition - splitStartPosition;
splits.add(new FileSplit(sequenceFile, splitStartPosition, splitSize, (String[]) null));
bytesRemaining -= splitSize;
splitStartPosition = splitEndPosition;
acc = 0L;
}
acc += key.get();
splitEndPosition = reader.getPosition();
}
} finally {
if (reader != null) {
reader.close();
}
}
if (bytesRemaining != 0) {
splits.add(new FileSplit(sequenceFile, splitStartPosition, bytesRemaining, (String[]) null));
}
return splits;
XXX There may be redundant location info available in the event.
We might consider extracting it from this event. Currently this
is redundant, but making this will add future-proofing.
LoggedTaskAttempt attempt = getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(), event.getAttemptId().toString());
if (attempt == null) {
return;
}
attempt.setResult(getPre21Value(event.getTaskStatus()));
attempt.setHostName(event.getHostname());
// XXX There may be redundant location info available in the event.
// We might consider extracting it from this event. Currently this
// is redundant, but making this will add future-proofing.
attempt.setFinishTime(event.getFinishTime());
attempt.incorporateCounters(((MapAttemptFinished) event.getDatum()).counters);
XXX There may be redundant location info available in the event.
We might consider extracting it from this event. Currently this
is redundant, but making this will add future-proofing.
LoggedTaskAttempt attempt = getOrMakeTaskAttempt(event.getTaskType(), event.getTaskId().toString(), event.getAttemptId().toString());
if (attempt == null) {
return;
}
attempt.setResult(getPre21Value(event.getTaskStatus()));
attempt.setHostName(event.getHostname());
// XXX There may be redundant location info available in the event.
// We might consider extracting it from this event. Currently this
// is redundant, but making this will add future-proofing.
attempt.setFinishTime(event.getFinishTime());
attempt.setShuffleFinished(event.getShuffleFinishTime());
attempt.setSortFinished(event.getSortFinishTime());
attempt.incorporateCounters(((ReduceAttemptFinished) event.getDatum()).counters);
TODO remove this once the deprecate APIs in LoggedJob are removed
SATD_ADDED
process(Properties)
public void process(Properties conf)
if (finalized) {
throw new IllegalStateException("JobBuilder.process(Properties conf) called after LoggedJob built");
}
// TODO remove this once the deprecate APIs in LoggedJob are removed
result.setQueue(extract(conf, JobConfPropertyNames.QUEUE_NAMES.getCandidates(), "default"));
result.setJobName(extract(conf, JobConfPropertyNames.JOB_NAMES.getCandidates(), null));
maybeSetHeapMegabytes(extractMegabytes(conf, JobConfPropertyNames.TASK_JAVA_OPTS_S.getCandidates()));
maybeSetJobMapMB(extractMegabytes(conf, JobConfPropertyNames.MAP_JAVA_OPTS_S.getCandidates()));
maybeSetJobReduceMB(extractMegabytes(conf, JobConfPropertyNames.REDUCE_JAVA_OPTS_S.getCandidates()));
this.jobConfigurationParameters = conf;
Now we need to generate the random numbers according to
the above distribution.
We create a lot of map tasks, each of which takes at least
one \"line\" of the distribution. (That is, a certain number
X is to be generated Y number of times.)
A map task emits Y key/val pairs. The val is X. The key
is a randomly-generated number.
The reduce task gets its input sorted by key. That is, sorted
in random order. It then emits a single line of text that
for the given values. It does not emit the key.
Because there's just one reduce task, we emit a single big
file of random numbers.
SATD_ADDED
launch()
public void launch() throws Exception
//
// Generate distribution of ints. This is the answer key.
//
JobConf conf = null;
// Check to get configuration and check if it is configured thro' Configured
// interface. This would happen when running testcase thro' command line.
if (getConf() == null) {
conf = new JobConf();
} else {
conf = new JobConf(getConf());
}
conf.setJarByClass(TestMapRed.class);
int countsToGo = counts;
int[] dist = new int[range];
for (int i = 0; i < range; i++) {
double avgInts = (1.0 * countsToGo) / (range - i);
dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian())));
countsToGo -= dist[i];
}
if (countsToGo > 0) {
dist[dist.length - 1] += countsToGo;
}
//
// Write the answer key to a file.
//
FileSystem fs = FileSystem.get(conf);
Path testdir = new Path("mapred.loadtest");
if (!fs.mkdirs(testdir)) {
throw new IOException("Mkdirs failed to create " + testdir.toString());
}
Path randomIns = new Path(testdir, "genins");
if (!fs.mkdirs(randomIns)) {
throw new IOException("Mkdirs failed to create " + randomIns.toString());
}
Path answerkey = new Path(randomIns, "answer.key");
SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, answerkey, IntWritable.class, IntWritable.class, SequenceFile.CompressionType.NONE);
try {
for (int i = 0; i < range; i++) {
out.append(new IntWritable(i), new IntWritable(dist[i]));
}
} finally {
out.close();
}
// printFiles(randomIns, conf);
//
// Now we need to generate the random numbers according to
// the above distribution.
//
// We create a lot of map tasks, each of which takes at least
// one "line" of the distribution. (That is, a certain number
// X is to be generated Y number of times.)
//
// A map task emits Y key/val pairs. The val is X. The key
// is a randomly-generated number.
//
// The reduce task gets its input sorted by key. That is, sorted
// in random order. It then emits a single line of text that
// for the given values. It does not emit the key.
//
// Because there's just one reduce task, we emit a single big
// file of random numbers.
//
Path randomOuts = new Path(testdir, "genouts");
fs.delete(randomOuts, true);
JobConf genJob = new JobConf(conf, TestMapRed.class);
FileInputFormat.setInputPaths(genJob, randomIns);
genJob.setInputFormat(SequenceFileInputFormat.class);
genJob.setMapperClass(RandomGenMapper.class);
FileOutputFormat.setOutputPath(genJob, randomOuts);
genJob.setOutputKeyClass(IntWritable.class);
genJob.setOutputValueClass(IntWritable.class);
genJob.setOutputFormat(TextOutputFormat.class);
genJob.setReducerClass(RandomGenReducer.class);
genJob.setNumReduceTasks(1);
JobClient.runJob(genJob);
// printFiles(randomOuts, conf);
//
// Next, we read the big file in and regenerate the
// original map. It's split into a number of parts.
// (That number is 'intermediateReduces'.)
//
// We have many map tasks, each of which read at least one
// of the output numbers. For each number read in, the
// map task emits a key/value pair where the key is the
// number and the value is "1".
//
// We have a single reduce task, which receives its input
// sorted by the key emitted above. For each key, there will
// be a certain number of "1" values. The reduce task sums
// these values to compute how many times the given key was
// emitted.
//
// The reduce task then emits a key/val pair where the key
// is the number in question, and the value is the number of
// times the key was emitted. This is the same format as the
// original answer key (except that numbers emitted zero times
// will not appear in the regenerated key.) The answer set
// is split into a number of pieces. A final MapReduce job
// will merge them.
//
// There's not really a need to go to 10 reduces here
// instead of 1. But we want to test what happens when
// you have multiple reduces at once.
//
int intermediateReduces = 10;
Path intermediateOuts = new Path(testdir, "intermediateouts");
fs.delete(intermediateOuts, true);
JobConf checkJob = new JobConf(conf, TestMapRed.class);
FileInputFormat.setInputPaths(checkJob, randomOuts);
checkJob.setInputFormat(TextInputFormat.class);
checkJob.setMapperClass(RandomCheckMapper.class);
FileOutputFormat.setOutputPath(checkJob, intermediateOuts);
checkJob.setOutputKeyClass(IntWritable.class);
checkJob.setOutputValueClass(IntWritable.class);
checkJob.setOutputFormat(MapFileOutputFormat.class);
checkJob.setReducerClass(RandomCheckReducer.class);
checkJob.setNumReduceTasks(intermediateReduces);
JobClient.runJob(checkJob);
// printFiles(intermediateOuts, conf);
//
// OK, now we take the output from the last job and
// merge it down to a single file. The map() and reduce()
// functions don't really do anything except reemit tuples.
// But by having a single reduce task here, we end up merging
// all the files.
//
Path finalOuts = new Path(testdir, "finalouts");
fs.delete(finalOuts, true);
JobConf mergeJob = new JobConf(conf, TestMapRed.class);
FileInputFormat.setInputPaths(mergeJob, intermediateOuts);
mergeJob.setInputFormat(SequenceFileInputFormat.class);
mergeJob.setMapperClass(MergeMapper.class);
FileOutputFormat.setOutputPath(mergeJob, finalOuts);
mergeJob.setOutputKeyClass(IntWritable.class);
mergeJob.setOutputValueClass(IntWritable.class);
mergeJob.setOutputFormat(SequenceFileOutputFormat.class);
mergeJob.setReducerClass(MergeReducer.class);
mergeJob.setNumReduceTasks(1);
JobClient.runJob(mergeJob);
// printFiles(finalOuts, conf);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path recomputedkey = new Path(finalOuts, "part-00000");
SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf);
int totalseen = 0;
try {
IntWritable key = new IntWritable();
IntWritable val = new IntWritable();
for (int i = 0; i < range; i++) {
if (dist[i] == 0) {
continue;
}
if (!in.next(key, val)) {
System.err.println("Cannot read entry " + i);
success = false;
break;
} else {
if (!((key.get() == i) && (val.get() == dist[i]))) {
System.err.println("Mismatch! Pos=" + key.get() + ", i=" + i + ", val=" + val.get() + ", dist[i]=" + dist[i]);
success = false;
}
totalseen += val.get();
}
}
if (success) {
if (in.next(key, val)) {
System.err.println("Unnecessary lines in recomputed key!");
success = false;
}
}
} finally {
in.close();
}
int originalTotal = 0;
for (int i = 0; i < dist.length; i++) {
originalTotal += dist[i];
}
System.out.println("Original sum: " + originalTotal);
System.out.println("Recomputed sum: " + totalseen);
//
// Write to "results" whether the test succeeded or not.
//
Path resultFile = new Path(testdir, "results");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile)));
try {
bw.write("Success=" + success + "\n");
System.out.println("Success=" + success);
} finally {
bw.close();
}
assertTrue("testMapRed failed", success);
fs.delete(testdir, true);
Checking if a Mapper.map within a Runnable has generated a
RuntimeException. If so we rethrow it to force an abort of the Map
operation thus keeping the semantics of the default
implementation.
SATD_ADDED
configure(JobConf)
public void configure(JobConf jobConf)
int numberOfThreads = jobConf.getInt(MultithreadedMapper.NUM_THREADS, 10);
if (LOG.isDebugEnabled()) {
LOG.debug("Configuring jobConf " + jobConf.getJobName() + " to use " + numberOfThreads + " threads");
}
this.job = jobConf;
// increment processed counter only if skipping feature is enabled
this.incrProcCount = SkipBadRecords.getMapperMaxSkipRecords(job) > 0 && SkipBadRecords.getAutoIncrMapperProcCount(job);
this.mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf);
// Creating a threadpool of the configured size to execute the Mapper
// map method in parallel.
executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, 0L, TimeUnit.MILLISECONDS, new BlockingArrayQueue(numberOfThreads));
Checking if a Mapper.map within a Runnable has generated an
IOException. If so we rethrow it to force an abort of the Map
operation thus keeping the semantics of the default
implementation.
SATD_ADDED
configure(JobConf)
public void configure(JobConf jobConf)
int numberOfThreads = jobConf.getInt(MultithreadedMapper.NUM_THREADS, 10);
if (LOG.isDebugEnabled()) {
LOG.debug("Configuring jobConf " + jobConf.getJobName() + " to use " + numberOfThreads + " threads");
}
this.job = jobConf;
// increment processed counter only if skipping feature is enabled
this.incrProcCount = SkipBadRecords.getMapperMaxSkipRecords(job) > 0 && SkipBadRecords.getAutoIncrMapperProcCount(job);
this.mapper = ReflectionUtils.newInstance(jobConf.getMapperClass(), jobConf);
// Creating a threadpool of the configured size to execute the Mapper
// map method in parallel.
executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, 0L, TimeUnit.MILLISECONDS, new BlockingArrayQueue(numberOfThreads));
checkNotSaved();
FSDirectory fsDir = sourceNamesystem.dir;
long startTime = now();
//
// Write out data
//
MessageDigest digester = MD5Hash.getDigester();
FileOutputStream fout = new FileOutputStream(newFile);
DigestOutputStream fos = new DigestOutputStream(fout, digester);
DataOutputStream out = new DataOutputStream(fos);
try {
out.writeInt(FSConstants.LAYOUT_VERSION);
// TODO bad dependency
out.writeInt(sourceNamesystem.getFSImage().getStorage().getNamespaceID());
out.writeLong(fsDir.rootDir.numItemsInTree());
out.writeLong(sourceNamesystem.getGenerationStamp());
// write compression info and set up compressed stream
out = compression.writeHeaderAndWrapStream(fos);
LOG.info("Saving image file " + newFile + " using " + compression);
byte[] byteStore = new byte[4 * FSConstants.MAX_PATH_LENGTH];
ByteBuffer strbuf = ByteBuffer.wrap(byteStore);
// save the root
FSImageSerialization.saveINode2Image(fsDir.rootDir, out);
// save the rest of the nodes
saveImage(strbuf, fsDir.rootDir, out);
// save files under construction
sourceNamesystem.saveFilesUnderConstruction(out);
sourceNamesystem.saveSecretManagerState(out);
strbuf = null;
out.flush();
fout.getChannel().force(true);
} finally {
out.close();
}
saved = true;
// set md5 of the saved image
savedDigest = new MD5Hash(digester.digest());
LOG.info("Image file of size " + newFile.length() + " saved in " + (now() - startTime) / 1000 + " seconds.");
Now we need to generate the random numbers according to
the above distribution.
We create a lot of map tasks, each of which takes at least
one \"line\" of the distribution. (That is, a certain number
X is to be generated Y number of times.)
A map task emits Y key/val pairs. The val is X. The key
is a randomly-generated number.
The reduce task gets its input sorted by key. That is, sorted
in random order. It then emits a single line of text that
for the given values. It does not emit the key.
Because there's just one reduce task, we emit a single big
file of random numbers.
SATD_ADDED
launch()
public static void launch() throws Exception
//
// Generate distribution of ints. This is the answer key.
//
int countsToGo = counts;
int[] dist = new int[range];
for (int i = 0; i < range; i++) {
double avgInts = (1.0 * countsToGo) / (range - i);
dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian())));
countsToGo -= dist[i];
}
if (countsToGo > 0) {
dist[dist.length - 1] += countsToGo;
}
//
// Write the answer key to a file.
//
FileSystem fs = FileSystem.get(conf);
Path testdir = new Path("mapred.loadtest");
if (!fs.mkdirs(testdir)) {
throw new IOException("Mkdirs failed to create directory " + testdir.toString());
}
Path randomIns = new Path(testdir, "genins");
if (!fs.mkdirs(randomIns)) {
throw new IOException("Mkdirs failed to create directory " + randomIns.toString());
}
Path answerkey = new Path(randomIns, "answer.key");
SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, answerkey, RecInt.class, RecInt.class, CompressionType.NONE);
try {
for (int i = 0; i < range; i++) {
RecInt k = new RecInt();
RecInt v = new RecInt();
k.setData(i);
v.setData(dist[i]);
out.append(k, v);
}
} finally {
out.close();
}
//
// Now we need to generate the random numbers according to
// the above distribution.
//
// We create a lot of map tasks, each of which takes at least
// one "line" of the distribution. (That is, a certain number
// X is to be generated Y number of times.)
//
// A map task emits Y key/val pairs. The val is X. The key
// is a randomly-generated number.
//
// The reduce task gets its input sorted by key. That is, sorted
// in random order. It then emits a single line of text that
// for the given values. It does not emit the key.
//
// Because there's just one reduce task, we emit a single big
// file of random numbers.
//
Path randomOuts = new Path(testdir, "genouts");
fs.delete(randomOuts, true);
JobConf genJob = new JobConf(conf, TestRecordMR.class);
FileInputFormat.setInputPaths(genJob, randomIns);
genJob.setInputFormat(SequenceFileInputFormat.class);
genJob.setMapperClass(RandomGenMapper.class);
FileOutputFormat.setOutputPath(genJob, randomOuts);
genJob.setOutputKeyClass(RecInt.class);
genJob.setOutputValueClass(RecString.class);
genJob.setOutputFormat(SequenceFileOutputFormat.class);
genJob.setReducerClass(RandomGenReducer.class);
genJob.setNumReduceTasks(1);
JobClient.runJob(genJob);
//
// Next, we read the big file in and regenerate the
// original map. It's split into a number of parts.
// (That number is 'intermediateReduces'.)
//
// We have many map tasks, each of which read at least one
// of the output numbers. For each number read in, the
// map task emits a key/value pair where the key is the
// number and the value is "1".
//
// We have a single reduce task, which receives its input
// sorted by the key emitted above. For each key, there will
// be a certain number of "1" values. The reduce task sums
// these values to compute how many times the given key was
// emitted.
//
// The reduce task then emits a key/val pair where the key
// is the number in question, and the value is the number of
// times the key was emitted. This is the same format as the
// original answer key (except that numbers emitted zero times
// will not appear in the regenerated key.) The answer set
// is split into a number of pieces. A final MapReduce job
// will merge them.
//
// There's not really a need to go to 10 reduces here
// instead of 1. But we want to test what happens when
// you have multiple reduces at once.
//
int intermediateReduces = 10;
Path intermediateOuts = new Path(testdir, "intermediateouts");
fs.delete(intermediateOuts, true);
JobConf checkJob = new JobConf(conf, TestRecordMR.class);
FileInputFormat.setInputPaths(checkJob, randomOuts);
checkJob.setInputFormat(SequenceFileInputFormat.class);
checkJob.setMapperClass(RandomCheckMapper.class);
FileOutputFormat.setOutputPath(checkJob, intermediateOuts);
checkJob.setOutputKeyClass(RecInt.class);
checkJob.setOutputValueClass(RecString.class);
checkJob.setOutputFormat(SequenceFileOutputFormat.class);
checkJob.setReducerClass(RandomCheckReducer.class);
checkJob.setNumReduceTasks(intermediateReduces);
JobClient.runJob(checkJob);
//
// OK, now we take the output from the last job and
// merge it down to a single file. The map() and reduce()
// functions don't really do anything except reemit tuples.
// But by having a single reduce task here, we end up merging
// all the files.
//
Path finalOuts = new Path(testdir, "finalouts");
fs.delete(finalOuts, true);
JobConf mergeJob = new JobConf(conf, TestRecordMR.class);
FileInputFormat.setInputPaths(mergeJob, intermediateOuts);
mergeJob.setInputFormat(SequenceFileInputFormat.class);
mergeJob.setMapperClass(MergeMapper.class);
FileOutputFormat.setOutputPath(mergeJob, finalOuts);
mergeJob.setOutputKeyClass(RecInt.class);
mergeJob.setOutputValueClass(RecInt.class);
mergeJob.setOutputFormat(SequenceFileOutputFormat.class);
mergeJob.setReducerClass(MergeReducer.class);
mergeJob.setNumReduceTasks(1);
JobClient.runJob(mergeJob);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path recomputedkey = new Path(finalOuts, "part-00000");
SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf);
int totalseen = 0;
try {
RecInt key = new RecInt();
RecInt val = new RecInt();
for (int i = 0; i < range; i++) {
if (dist[i] == 0) {
continue;
}
if (!in.next(key, val)) {
System.err.println("Cannot read entry " + i);
success = false;
break;
} else {
if (!((key.getData() == i) && (val.getData() == dist[i]))) {
System.err.println("Mismatch! Pos=" + key.getData() + ", i=" + i + ", val=" + val.getData() + ", dist[i]=" + dist[i]);
success = false;
}
totalseen += val.getData();
}
}
if (success) {
if (in.next(key, val)) {
System.err.println("Unnecessary lines in recomputed key!");
success = false;
}
}
} finally {
in.close();
}
int originalTotal = 0;
for (int i = 0; i < dist.length; i++) {
originalTotal += dist[i];
}
System.out.println("Original sum: " + originalTotal);
System.out.println("Recomputed sum: " + totalseen);
//
// Write to "results" whether the test succeeded or not.
//
Path resultFile = new Path(testdir, "results");
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile)));
try {
bw.write("Success=" + success + "\n");
System.out.println("Success=" + success);
} finally {
bw.close();
}
fs.delete(testdir, true);
Since we're creating a new UserGroupInformation here, we know that no
future RPC proxies will be able to re-use the same connection. And
usages of this proxy tend to be one-off calls.
This is a temporary fix: callers should really achieve this by using
RPC.stopProxy() on the resulting object, but this is currently not
working in trunk. See the discussion on HDFS-1965.
InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getHost() + ":" + datanodeid.getIpcPort());
if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
ClientDatanodeProtocol.LOG.debug("ClientDatanodeProtocol addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
UserGroupInformation ticket = UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
ticket.addToken(locatedBlock.getBlockToken());
return (ClientDatanodeProtocol) RPC.getProxy(ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
if (!conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY, DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT)) {
throw new IOException("The option " + DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY + " is set to false for this filesystem, so it " + "cannot be formatted. You will need to set " + DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY + " parameter " + "to true in order to format this filesystem");
}
Collection dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
Collection editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf);
for (Iterator it = dirsToFormat.iterator(); it.hasNext(); ) {
File curDir = new File(it.next().getPath());
// Its alright for a dir not to exist, or to exist (properly accessible)
// and be completely empty.
if (!curDir.exists() || (curDir.isDirectory() && FileUtil.listFiles(curDir).length == 0))
continue;
if (isConfirmationNeeded) {
if (!confirmPrompt("Re-format filesystem in " + curDir + " ?")) {
System.err.println("Format aborted in " + curDir);
return true;
}
}
}
// if clusterID is not provided - see if you can find the current one
String clusterId = StartupOption.FORMAT.getClusterId();
if (clusterId == null || clusterId.equals("")) {
// Generate a new cluster id
clusterId = NNStorage.newClusterID();
}
System.out.println("Formatting using clusterid: " + clusterId);
FSImage fsImage = new FSImage(dirsToFormat, editDirsToFormat);
FSNamesystem nsys = new FSNamesystem(fsImage, conf);
nsys.dir.fsImage.getStorage().format(clusterId);
return false;
Private Distributed cache will always be stored under
mapre.local.dir/taskTracker//distcache
Checking for username directory to check if it has the
proper permissions
SATD_ADDED
testDistributedCache()
public void testDistributedCache() throws Exception
Configuration conf = new Configuration(cluster.getConf());
JTProtocol wovenClient = cluster.getJTClient().getProxy();
// This counter will check for count of a loop,
// which might become infinite.
int count = 0;
SleepJob job = new SleepJob();
job.setConf(conf);
Job slpJob = job.createJob(5, 1, 1000, 1000, 100, 100);
DistributedCache.createSymlink(conf);
URI uri = URI.create(uriPath);
DistributedCache.addCacheFile(uri, conf);
JobConf jconf = new JobConf(conf);
// Controls the job till all verification is done
FinishTaskControlAction.configureControlActionForJob(conf);
// Submitting the job
slpJob.submit();
RunningJob rJob = cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));
JobStatus[] jobStatus = client.getAllJobs();
String userName = jobStatus[0].getUsername();
TTClient tClient = null;
JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
LOG.info("jInfo is :" + jInfo);
// Assert if jobInfo is null
Assert.assertNotNull("jobInfo is null", jInfo);
// Wait for the job to start running.
count = 0;
while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
UtilsForTests.waitFor(10000);
count++;
jInfo = wovenClient.getJobInfo(rJob.getID());
// If the count goes beyond a point, then Assert; This is to avoid
// infinite loop under unforeseen circumstances.
if (count > 10) {
Assert.fail("job has not reached running state for more than" + "100 seconds. Failing at this point");
}
}
LOG.info("job id is :" + rJob.getID().toString());
TaskInfo[] taskInfos = cluster.getJTClient().getProxy().getTaskInfo(rJob.getID());
boolean distCacheFileIsFound;
for (TaskInfo taskInfo : taskInfos) {
distCacheFileIsFound = false;
String[] taskTrackers = taskInfo.getTaskTrackers();
for (String taskTracker : taskTrackers) {
// Getting the exact FQDN of the tasktracker from
// the tasktracker string.
taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
tClient = cluster.getTTClient(taskTracker);
String[] localDirs = tClient.getMapredLocalDirs();
int distributedFileCount = 0;
String localDirOnly = null;
boolean FileNotPresentForThisDirectoryPath = false;
// Go to every single path
for (String localDir : localDirs) {
FileNotPresentForThisDirectoryPath = false;
localDirOnly = localDir;
// Public Distributed cache will always be stored under
// mapred.local.dir/tasktracker/archive
localDirOnly = localDir + Path.SEPARATOR + TaskTracker.SUBDIR + Path.SEPARATOR + userName;
// Private Distributed cache will always be stored under
// mapre.local.dir/taskTracker//distcache
// Checking for username directory to check if it has the
// proper permissions
localDir = localDir + Path.SEPARATOR + TaskTracker.getPrivateDistributedCacheDir(userName);
FileStatus fileStatusMapredLocalDirUserName = null;
try {
fileStatusMapredLocalDirUserName = tClient.getFileStatus(localDirOnly, true);
} catch (Exception e) {
LOG.info("LocalDirOnly :" + localDirOnly + " not found");
FileNotPresentForThisDirectoryPath = true;
}
// File will only be stored under one of the mapred.lcoal.dir
// If other paths were hit, just continue
if (FileNotPresentForThisDirectoryPath)
continue;
Path pathMapredLocalDirUserName = fileStatusMapredLocalDirUserName.getPath();
FsPermission fsPermMapredLocalDirUserName = fileStatusMapredLocalDirUserName.getPermission();
Assert.assertTrue("Directory Permission is not 700", fsPermMapredLocalDirUserName.equals(new FsPermission("700")));
// Get file status of all the directories
// and files under that path.
FileStatus[] fileStatuses = tClient.listStatus(localDir, true, true);
for (FileStatus fileStatus : fileStatuses) {
Path path = fileStatus.getPath();
LOG.info("path is :" + path.toString());
// Checking if the received path ends with
// the distributed filename
distCacheFileIsFound = (path.toString()).endsWith(distributedFileName);
// If file is found, check for its permission.
// Since the file is found break out of loop
if (distCacheFileIsFound) {
LOG.info("PATH found is :" + path.toString());
distributedFileCount++;
String filename = path.getName();
FsPermission fsPerm = fileStatus.getPermission();
Assert.assertTrue("File Permission is not 777", fsPerm.equals(new FsPermission("777")));
}
}
}
LOG.info("Distributed File count is :" + distributedFileCount);
if (distributedFileCount > 1) {
Assert.fail("The distributed cache file is more than one");
} else if (distributedFileCount < 1)
Assert.fail("The distributed cache file is less than one");
if (!distCacheFileIsFound) {
Assert.assertEquals("The distributed cache file does not exist", distCacheFileIsFound, false);
}
}
// Allow the job to continue through MR control job.
for (TaskInfo taskInfoRemaining : taskInfos) {
FinishTaskControlAction action = new FinishTaskControlAction(TaskID.downgrade(taskInfoRemaining.getTaskID()));
Collection tts = cluster.getTTClients();
for (TTClient cli : tts) {
cli.getProxy().sendAction(action);
}
}
// Killing the job because all the verification needed
// for this testcase is completed.
rJob.killJob();
}
Validate if DistributedCacheEmulator can handle a JobStory with out
Distributed Cache files properly.
SATD_ADDED
testDistCacheFilesConfiguration()
public void testDistCacheFilesConfiguration() throws IOException
Configuration conf = new Configuration();
JobConf jobConf = GridmixTestUtils.mrCluster.createJobConf(new JobConf(conf));
Path ioPath = new Path("testDistCacheEmulationConfigurability").makeQualified(GridmixTestUtils.dfs);
FileSystem fs = FileSystem.get(jobConf);
FileSystem.mkdirs(fs, ioPath, new FsPermission((short) 0777));
// default config
dce = createDistributedCacheEmulator(jobConf, ioPath, false);
assertTrue("Default configuration of " + DistributedCacheEmulator.GRIDMIX_EMULATE_DISTRIBUTEDCACHE + " is wrong.", dce.shouldEmulateDistCacheLoad());
// Validate if DistributedCacheEmulator can handle a JobStory with out
// Distributed Cache files properly.
validateJobConfWithOutDCFiles(conf, jobConf);
// Validate if Gridmix can configure dist cache files properly if there are
// HDFS-based dist cache files and localFS-based dist cache files in trace
// for a job. Set old config properties and validate.
validateJobConfWithDCFiles(conf, jobConf);
// Use new JobConf as JobStory conf and check if configureDistCacheFiles()
// doesn't throw NPE when there are dist cache files set but visibilities
// are not set.
validateWithOutVisibilities();
Verify if correct exit code is seen when -generate option is missing and
distributed cache files are missing in the expected path.
SATD_ADDED
testSetupGenerateDistCacheData()
public void testSetupGenerateDistCacheData() throws IOException, InterruptedException
long[] sortedFileSizes = new long[5];
JobConf jobConf = runSetupGenerateDistCacheData(true, sortedFileSizes);
validateSetupGenDC(jobConf, sortedFileSizes);
// Verify if correct exit code is seen when -generate option is missing and
// distributed cache files are missing in the expected path.
runSetupGenerateDistCacheData(false, sortedFileSizes);
Path distCachePath = dce.getDistributedCacheDir();
String filesListFile = jobConf.get(GenerateDistCacheData.GRIDMIX_DISTCACHE_FILE_LIST);
FileSystem fs = FileSystem.get(jobConf);
// Validate the existence of Distributed Cache files list file directly
// under distributed cache directory
Path listFile = new Path(filesListFile);
assertTrue("Path of Distributed Cache files list file is wrong.", distCachePath.equals(listFile.getParent().makeQualified(fs)));
// Delete the dist cache files list file
assertTrue("Failed to delete distributed Cache files list file " + listFile, fs.delete(listFile));
List fileSizes = new ArrayList();
for (long size : sortedFileSizes) {
fileSizes.add(size);
}
// validate dist cache files after deleting the 'files list file'
validateDistCacheFiles(fileSizes, distCachePath);
our target rate is in terms of the maximum number of sample splits,
but we accept the possibility of sampling additional splits to hit
the target sample keyset
initialize buffer to the best guess size:
'chunksPerPacket' calculation here should match the same
calculation in DFSClient to make the guess accurate.
We should never receive a duplicate success/failure/killed
status update for the same taskid! This is a safety check,
and is addressed better at the TaskTracker to ensure this.
desired range starts after beginning of this har block
fix offset to beginning of relevant range (relative to desired file)
SATD_ADDED
initialize(URI, Configuration)
public void initialize(URI name, Configuration conf) throws IOException
// decode the name
URI underLyingURI = decodeHarURI(name, conf);
// we got the right har Path- now check if this is
// truly a har filesystem
Path harPath = archivePath(new Path(name.getScheme(), name.getAuthority(), name.getPath()));
if (harPath == null) {
throw new IOException("Invalid path for the Har Filesystem. " + name.toString());
}
if (fs == null) {
fs = FileSystem.get(underLyingURI, conf);
}
uri = harPath.toUri();
archivePath = new Path(uri.getPath());
harAuth = getHarAuth(underLyingURI);
// check for the underlying fs containing
// the index file
Path masterIndexPath = new Path(archivePath, "_masterindex");
Path archiveIndexPath = new Path(archivePath, "_index");
if (!fs.exists(masterIndexPath) || !fs.exists(archiveIndexPath)) {
throw new IOException("Invalid path for the Har Filesystem. " + "No index file in " + harPath);
}
metadata = harMetaCache.get(uri);
if (metadata != null) {
FileStatus mStat = fs.getFileStatus(masterIndexPath);
FileStatus aStat = fs.getFileStatus(archiveIndexPath);
if (mStat.getModificationTime() != metadata.getMasterIndexTimestamp() || aStat.getModificationTime() != metadata.getArchiveIndexTimestamp()) {
// the archive has been overwritten since we last read it
// remove the entry from the meta data cache
metadata = null;
harMetaCache.remove(uri);
}
}
if (metadata == null) {
metadata = new HarMetaData(fs, masterIndexPath, archiveIndexPath);
metadata.parseMetaData();
harMetaCache.put(uri, metadata);
}
public static boolean injectCriteria(String klassName)
boolean trigger = false;
// TODO fix this: make it more sophisticated!!!
if (generator.nextFloat() < getProbability(klassName)) {
trigger = true;
}
return trigger;
Set new default probability if specified through a system.property
If neither is specified set default probability to DEFAULT_PROB
SATD_ADDED
injectCriteria(String)
public static boolean injectCriteria(String klassName)
boolean trigger = false;
// TODO fix this: make it more sophisticated!!!
if (generator.nextFloat() < getProbability(klassName)) {
trigger = true;
}
return trigger;
StringBuilder split = new StringBuilder();
int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split);
// clear the buffer
split.setLength(0);
if (next >= 0) {
// move over '('
++next;
next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split);
if (next >= 0) {
// move over ')'
++next;
index.set(next);
// found a block
return split.toString();
} else {
throw new ParseException("Unexpected end of block", next);
}
}
// found nothing
return null;
StringBuilder split = new StringBuilder();
int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, index.get(), split);
// clear the buffer
split.setLength(0);
if (next >= 0) {
// move over '('
++next;
next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, next, split);
if (next >= 0) {
// move over ')'
++next;
index.set(next);
// found a block
return split.toString();
} else {
throw new ParseException("Unexpected end of block", next);
}
}
// found nothing
return null;
public void testCumulativeCpuUsageEmulatorPlugin() throws Exception
Configuration conf = new Configuration();
long targetCpuUsage = 1000L;
int unitCpuUsage = 50;
// fake progress indicator
FakeProgressive fakeProgress = new FakeProgressive();
// fake cpu usage generator
FakeCpuUsageEmulatorCore fakeCore = new FakeCpuUsageEmulatorCore();
fakeCore.setUnitUsage(unitCpuUsage);
// a cumulative cpu usage emulator with fake core
CumulativeCpuUsageEmulatorPlugin cpuPlugin = new CumulativeCpuUsageEmulatorPlugin(fakeCore);
// test with invalid or missing resource usage value
ResourceUsageMetrics invalidUsage = createMetrics(0);
cpuPlugin.initialize(conf, invalidUsage, null, null);
// test if disabled cpu emulation plugin's emulate() call is a no-operation
// this will test if the emulation plugin is disabled or not
int numCallsPre = fakeCore.getNumCalls();
long cpuUsagePre = fakeCore.getCpuUsage();
cpuPlugin.emulate();
int numCallsPost = fakeCore.getNumCalls();
long cpuUsagePost = fakeCore.getCpuUsage();
// test if no calls are made cpu usage emulator core
assertEquals("Disabled cumulative CPU usage emulation plugin works!", numCallsPre, numCallsPost);
// test if no calls are made cpu usage emulator core
assertEquals("Disabled cumulative CPU usage emulation plugin works!", cpuUsagePre, cpuUsagePost);
// test with valid resource usage value
ResourceUsageMetrics metrics = createMetrics(targetCpuUsage);
// fake monitor
ResourceCalculatorPlugin monitor = new FakeResourceUsageMonitor(fakeCore);
// test with default emulation interval
testEmulationAccuracy(conf, fakeCore, monitor, metrics, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage);
// test with custom value for emulation interval of 20%
conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_FREQUENCY, 0.2F);
testEmulationAccuracy(conf, fakeCore, monitor, metrics, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage);
// test if emulation interval boundary is respected (unit usage = 1)
// test the case where the current progress is less than threshold
// initialize
fakeProgress = new FakeProgressive();
fakeCore.reset();
fakeCore.setUnitUsage(1);
conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_FREQUENCY, 0.25F);
cpuPlugin.initialize(conf, metrics, monitor, fakeProgress);
// take a snapshot after the initialization
long initCpuUsage = monitor.getCumulativeCpuTime();
long initNumCalls = fakeCore.getNumCalls();
// test with 0 progress
testEmulationBoundary(0F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 0 progress]");
// test with 24% progress
testEmulationBoundary(0.24F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 24% progress]");
// test with 25% progress
// target = 1000ms, target emulation at 25% = 250ms,
// weighed target = 1000 * 0.25^4 (we are using progress^4 as the weight)
// ~ 4
// but current usage = init-usage = 100, hence expected = 100
testEmulationBoundary(0.25F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[op, 25% progress]");
// test with 80% progress
// target = 1000ms, target emulation at 80% = 800ms,
// weighed target = 1000 * 0.25^4 (we are using progress^4 as the weight)
// ~ 410
// current-usage = init-usage = 100, hence expected-usage = 410
testEmulationBoundary(0.80F, fakeCore, fakeProgress, cpuPlugin, 410, 410, "[op, 80% progress]");
// now test if the final call with 100% progress ramps up the CPU usage
testEmulationBoundary(1F, fakeCore, fakeProgress, cpuPlugin, targetCpuUsage, targetCpuUsage, "[op, 100% progress]");
// test if emulation interval boundary is respected (unit usage = 50)
// test the case where the current progress is less than threshold
// initialize
fakeProgress = new FakeProgressive();
fakeCore.reset();
fakeCore.setUnitUsage(unitCpuUsage);
conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_FREQUENCY, 0.40F);
cpuPlugin.initialize(conf, metrics, monitor, fakeProgress);
// take a snapshot after the initialization
initCpuUsage = monitor.getCumulativeCpuTime();
initNumCalls = fakeCore.getNumCalls();
// test with 0 progress
testEmulationBoundary(0F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 0 progress]");
// test with 39% progress
testEmulationBoundary(0.39F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 39% progress]");
// test with 40% progress
// target = 1000ms, target emulation at 40% = 4000ms,
// weighed target = 1000 * 0.40^4 (we are using progress^4 as the weight)
// ~ 26
// current-usage = init-usage = 100, hence expected-usage = 100
testEmulationBoundary(0.40F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[op, 40% progress]");
// test with 90% progress
// target = 1000ms, target emulation at 90% = 900ms,
// weighed target = 1000 * 0.90^4 (we are using progress^4 as the weight)
// ~ 657
// current-usage = init-usage = 100, hence expected-usage = 657 but
// the fake-core increases in steps of 50, hence final target = 700
testEmulationBoundary(0.90F, fakeCore, fakeProgress, cpuPlugin, 700, 700 / unitCpuUsage, "[op, 90% progress]");
// now test if the final call with 100% progress ramps up the CPU usage
testEmulationBoundary(1F, fakeCore, fakeProgress, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage, "[op, 100% progress]");
Remove all policies where srcPath <= currentSrcPath or
matchingPrefixLength is < length(currentSrcPath)
The policies remaining are the only ones that could better
select a file chosen by the current policy.
We need this semaphore to block when the number of running workitems
is equal to the number of threads. FixedThreadPool limits the number
of threads, but not the queue size. This way we will limit the memory
usage.
public void emulate() throws IOException, InterruptedException
if (enabled) {
float currentProgress = progress.getProgress();
if (lastSeenProgress < currentProgress && ((currentProgress - lastSeenProgress) >= emulationInterval || currentProgress == 1)) {
// Estimate the final cpu usage
//
// Consider the following
// Cl/Cc/Cp : Last/Current/Projected Cpu usage
// Pl/Pc/Pp : Last/Current/Projected progress
// Then
// (Cp-Cc)/(Pp-Pc) = (Cc-Cl)/(Pc-Pl)
// Solving this for Cp, we get
// Cp = Cc + (1-Pc)*(Cc-Cl)/Pc-Pl)
// Note that (Cc-Cl)/(Pc-Pl) is termed as 'rate' in the following
// section
long currentCpuUsage = monitor.getProcResourceValues().getCumulativeCpuTime();
// estimate the cpu usage rate
float rate = (currentCpuUsage - lastSeenCpuUsageCpuUsage) / (currentProgress - lastSeenProgress);
long projectedUsage = currentCpuUsage + (long) ((1 - currentProgress) * rate);
if (projectedUsage < targetCpuUsage) {
// determine the correction factor between the current usage and the
// expected usage and add some weight to the target
long currentWeighedTarget = (long) (targetCpuUsage * getWeightForProgressInterval(currentProgress));
while (monitor.getProcResourceValues().getCumulativeCpuTime() < currentWeighedTarget) {
emulatorCore.compute();
// sleep for 100ms
try {
Thread.sleep(100);
} catch (InterruptedException ie) {
String message = "CumulativeCpuUsageEmulatorPlugin got interrupted. Exiting.";
throw new RuntimeException(message);
}
}
}
// set the last seen progress
lastSeenProgress = progress.getProgress();
// set the last seen usage
lastSeenCpuUsageCpuUsage = monitor.getProcResourceValues().getCumulativeCpuTime();
}
}
final long startTime = System.currentTimeMillis();
try {
String msg = "FSCK started by " + UserGroupInformation.getCurrentUser() + " from " + remoteAddress + " for path " + path + " at " + new Date();
LOG.info(msg);
out.println(msg);
namenode.getNamesystem().logFsckEvent(path, remoteAddress);
final HdfsFileStatus file = namenode.getFileInfo(path);
if (file != null) {
if (showCorruptFileBlocks) {
listCorruptFileBlocks();
return;
}
Result res = new Result(conf);
check(path, file, res);
out.println(res);
out.println(" Number of data-nodes:\t\t" + totalDatanodes);
out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks());
out.println("FSCK ended at " + new Date() + " in " + (System.currentTimeMillis() - startTime + " milliseconds"));
// DFSck client scans for the string HEALTHY/CORRUPT to check the status
// of file system and return appropriate code. Changing the output
// string might break testcases. Also note this must be the last line
// of the report.
if (res.isHealthy()) {
out.print("\n\nThe filesystem under path '" + path + "' " + HEALTHY_STATUS);
} else {
out.print("\n\nThe filesystem under path '" + path + "' " + CORRUPT_STATUS);
}
} else {
out.print("\n\nPath '" + path + "' " + NONEXISTENT_STATUS);
}
} catch (Exception e) {
String errMsg = "Fsck on path '" + path + "' " + FAILURE_STATUS;
LOG.warn(errMsg, e);
out.println("FSCK ended at " + new Date() + " in " + (System.currentTimeMillis() - startTime + " milliseconds"));
out.println(e.getMessage());
out.print("\n\n" + errMsg);
} finally {
out.close();
}
we want some kind of exponential growth function that gives less weight
on lower progress boundaries but high (exact emulation) near progress
value of 1.
so here is how the current growth function looks like
progress weight
0.1 0.0001
0.2 0.0016
0.3 0.0081
0.4 0.0256
0.5 0.0625
0.6 0.1296
0.7 0.2401
0.8 0.4096
0.9 0.6561
1.0 1.000
// we want some kind of exponential growth function that gives less weight
// on lower progress boundaries but high (exact emulation) near progress
// value of 1.
// so here is how the current growth function looks like
// progress weight
// 0.1 0.0001
// 0.2 0.0016
// 0.3 0.0081
// 0.4 0.0256
// 0.5 0.0625
// 0.6 0.1296
// 0.7 0.2401
// 0.8 0.4096
// 0.9 0.6561
// 1.0 1.000
return progress * progress * progress * progress;
compute the 1% of the total CPU usage desired
TODO Make this configurable
SATD_ADDED
calibrate(ResourceCalculatorPlugin, long)
public void calibrate(ResourceCalculatorPlugin monitor, long totalCpuUsage)
long initTime = monitor.getProcResourceValues().getCumulativeCpuTime();
long defaultLoopSize = 0;
long finalTime = initTime;
// TODO Make this configurable
while (finalTime - initTime < 100) {
// 100 ms
++defaultLoopSize;
// perform unit computation
performUnitComputation();
finalTime = monitor.getProcResourceValues().getCumulativeCpuTime();
}
long referenceRuntime = finalTime - initTime;
// time for one loop = (final-time - init-time) / total-loops
float timePerLoop = ((float) referenceRuntime) / defaultLoopSize;
// compute the 1% of the total CPU usage desired
// TODO Make this configurable
long onePercent = totalCpuUsage / 100;
// num-iterations for 1% = (total-desired-usage / 100) / time-for-one-loop
numIterations = Math.max(1, (int) ((float) onePercent / timePerLoop));
System.out.println("Calibration done. Basic computation runtime : " + timePerLoop + " milliseconds. Optimal number of iterations (1%): " + numIterations);
public void calibrate(ResourceCalculatorPlugin monitor, long totalCpuUsage)
long initTime = monitor.getProcResourceValues().getCumulativeCpuTime();
long defaultLoopSize = 0;
long finalTime = initTime;
// TODO Make this configurable
while (finalTime - initTime < 100) {
// 100 ms
++defaultLoopSize;
// perform unit computation
performUnitComputation();
finalTime = monitor.getProcResourceValues().getCumulativeCpuTime();
}
long referenceRuntime = finalTime - initTime;
// time for one loop = (final-time - init-time) / total-loops
float timePerLoop = ((float) referenceRuntime) / defaultLoopSize;
// compute the 1% of the total CPU usage desired
// TODO Make this configurable
long onePercent = totalCpuUsage / 100;
// num-iterations for 1% = (total-desired-usage / 100) / time-for-one-loop
numIterations = Math.max(1, (int) ((float) onePercent / timePerLoop));
System.out.println("Calibration done. Basic computation runtime : " + timePerLoop + " milliseconds. Optimal number of iterations (1%): " + numIterations);
TODO can this be configurable too. Users/emulators should be able to
pick and choose what MATH operations to run.
Example :
BASIC : ADD, SUB, MUL, DIV
ADV : SQRT, SIN, COSIN..
COMPO : (BASIC/ADV)*
Also define input generator. For now we can use the random number
generator. Later this can be changed to accept multiple sources.
SATD_ADDED
performUnitComputation()
protected void performUnitComputation()
// TODO can this be configurable too. Users/emulators should be able to
// pick and choose what MATH operations to run.
// Example :
// BASIC : ADD, SUB, MUL, DIV
// ADV : SQRT, SIN, COSIN..
// COMPO : (BASIC/ADV)*
// Also define input generator. For now we can use the random number
// generator. Later this can be changed to accept multiple sources.
int randomData = random.nextInt();
int randomDataCube = randomData * randomData * randomData;
double randomDataCubeRoot = Math.cbrt(randomData);
returnValue = Math.log(Math.tan(randomDataCubeRoot * Math.exp(randomDataCube)) * Math.sqrt(randomData));
this is an unexpected condition, so dump the whole exception since
it's probably a nasty internal error where the backtrace would be
useful
this is an unexpected condition, so dump the whole exception since
it's probably a nasty internal error where the backtrace would be
useful
FILE_PATH_CHANGED
displayError(Exception)
public void displayError(Exception e)
// build up a list of exceptions that occurred
exceptions.add(e);
String errorMessage = e.getLocalizedMessage();
if (errorMessage == null) {
// this is an unexpected condition, so dump the whole exception since
// it's probably a nasty internal error where the backtrace would be
// useful
errorMessage = StringUtils.stringifyException(e);
LOG.debug(errorMessage);
} else {
errorMessage = errorMessage.split("\n", 2)[0];
}
displayError(errorMessage);
The following if clause handles the following case:
Assume the following scenario in BZip2 compressed stream where
. represent compressed data.
.....[48 bit Block].....[48 bit Block].....[48 bit Block]...
........................[47 bits][1 bit].....[48 bit Block]...
................................^[Assume a Byte alignment here]
........................................^^[current position of stream]
.....................^^[We go back 10 Bytes in stream and find a Block marker]
........................................^^[We align at wrong position!]
...........................................................^^[While this pos is correct]
The following if clause handles the following case:
Assume the following scenario in BZip2 compressed stream where
. represent compressed data.
.....[48 bit Block].....[48 bit Block].....[48 bit Block]...
........................[47 bits][1 bit].....[48 bit Block]...
................................^[Assume a Byte alignment here]
........................................^^[current position of stream]
.....................^^[We go back 10 Bytes in stream and find a Block marker]
........................................^^[We align at wrong position!]
...........................................................^^[While this pos is correct]
FILE_PATH_CHANGED
createOutputStream(OutputStream)
public CompressionOutputStream createOutputStream(OutputStream out) throws IOException
public synchronized void refresh(Configuration conf, PolicyProvider provider)
// Get the system property 'hadoop.policy.file'
String policyFile = System.getProperty("hadoop.policy.file", HADOOP_POLICY_FILE);
// Make a copy of the original config, and load the policy file
Configuration policyConf = new Configuration(conf);
policyConf.addResource(policyFile);
final Map, AccessControlList> newAcls = new IdentityHashMap, AccessControlList>();
// Parse the config file
Service[] services = provider.getServices();
if (services != null) {
for (Service service : services) {
AccessControlList acl = new AccessControlList(policyConf.get(service.getServiceKey(), AccessControlList.WILDCARD_ACL_VALUE));
newAcls.put(service.getProtocol(), acl);
}
}
// Flip to the newly parsed permissions
protocolToAcl = newAcls;
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFsFileStatus that works around.
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFsFileStatus that works around.
FILE_PATH_CHANGED
readOnlyMountTable(String, String)
static AccessControlException readOnlyMountTable(final String operation, final String p)
return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
public boolean mkdirs(Path dir, FsPermission permission) throws AccessControlException, FileAlreadyExistsException
if (theInternalDir.isRoot & dir == null) {
throw new FileAlreadyExistsException("/ already exits");
}
// Note dir starts with /
if (theInternalDir.children.containsKey(dir.toString().substring(1))) {
// this is the stupid semantics of FileSystem
return true;
}
throw readOnlyMountTable("mkdirs", dir);
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFileSystemFileStatus that
works around.
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFileSystemFileStatus that
works around.
FILE_PATH_CHANGED
readOnlyMountTable(String, String)
static AccessControlException readOnlyMountTable(final String operation, final String p)
return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
public void testDeserialization() throws IOException
// Create a test BlockLocation
String[] names = { "one", "two" };
String[] hosts = { "three", "four" };
String[] topologyPaths = { "five", "six" };
long offset = 25l;
long length = 55l;
BlockLocation bl = new BlockLocation(names, hosts, topologyPaths, offset, length);
DataOutputBuffer dob = new DataOutputBuffer();
// Serialize it
try {
bl.write(dob);
} catch (IOException e) {
fail("Unable to serialize data: " + e.getMessage());
}
byte[] bytes = dob.getData();
DataInput da = new DataInputStream(new ByteArrayInputStream(bytes));
// Try to re-create the BlockLocation the same way as is done during
// deserialization
BlockLocation bl2 = new BlockLocation();
try {
bl2.readFields(da);
} catch (IOException e) {
fail("Unable to deserialize BlockLocation: " + e.getMessage());
}
// Check that we got back what we started with
verifyDeserialization(bl2.getHosts(), hosts);
verifyDeserialization(bl2.getNames(), names);
verifyDeserialization(bl2.getTopologyPaths(), topologyPaths);
assertEquals(bl2.getOffset(), offset);
assertEquals(bl2.getLength(), length);
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
if (name == null) {
LOG.warn("Metric was emitted with no name.");
return;
} else if (value == null) {
LOG.warn("Metric name " + name + " was emitted with a null value.");
return;
} else if (type == null) {
LOG.warn("Metric name " + name + ", value " + value + " has no type.");
return;
}
LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName);
String units = getUnits(name);
if (units == null) {
LOG.warn("Metric name " + name + ", value " + value + " had 'null' units");
units = "";
}
int slope = getSlope(name);
int tmax = getTmax(name);
int dmax = getDmax(name);
offset = 0;
String groupName = name.substring(0, name.lastIndexOf("."));
// The following XDR recipe was done through a careful reading of
// gm_protocol.x in Ganglia 3.1 and carefully examining the output of
// the gmetric utility with strace.
// First we send out a metadata message
// metric_id = metadata_msg
xdr_int(128);
// hostname
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// metric type
xdr_string(type);
// metric name
xdr_string(name);
// units
xdr_string(units);
// slope
xdr_int(slope);
// tmax, the maximum time between metrics
xdr_int(tmax);
// dmax, the maximum data value
xdr_int(dmax);
xdr_int(1);
/*Num of the entries in extra_value field for
Ganglia 3.1.x*/
xdr_string("GROUP");
/*Group attribute*/
xdr_string(groupName);
/*Group value*/
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
// Now we send out a message with the actual value.
// Technically, we only need to send out the metadata message once for
// each metric, but I don't want to have to record which metrics we did and
// did not send.
offset = 0;
// we are sending a string value
xdr_int(133);
// hostName
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// format field
xdr_string("%s");
// metric value
xdr_string(value);
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
public static int writeString(DataOutput out, String s) throws IOException
if (s.length() > 0xffff / 3) {
// maybe too long
LOG.warn("truncating long string: " + s.length() + " chars, starting with " + s.substring(0, 20));
s = s.substring(0, 0xffff / 3);
}
int len = utf8Length(s);
if (// double-check length
len > 0xffff)
throw new IOException("string too long!");
out.writeShort(len);
writeChars(out, s, 0, s.length());
return len;
Some filesystem like HDFS ignore the \"x\" bit if the permission.
Others like localFs does not.
Override the method below if the file system being tested masks our
certain bits for file masks.
Some filesystem like HDFS ignore the \"x\" bit if the permission.
Others like localFs does not.
Override the method below if the file system being tested masks our
certain bits for file masks.
//
// Done with local copy
//
backupStream.close();
//
// Send it to S3
//
// TODO: Use passed in Progressable to report progress.
nextBlockOutputStream();
store.storeBlock(nextBlock, backupFile);
internalClose();
//
// Delete local backup, start new one
//
boolean b = backupFile.delete();
if (!b) {
LOG.warn("Ignoring failed delete");
}
backupFile = newBackupFile();
backupStream = new FileOutputStream(backupFile);
bytesWrittenToBlock = 0;
public boolean reportChecksumFailure(Path p, FSDataInputStream in, long inPos, FSDataInputStream sums, long sumsPos)
try {
// canonicalize f
File f = ((RawLocalFileSystem) fs).pathToFile(p).getCanonicalFile();
// find highest writable parent dir of f on the same device
String device = new DF(f, getConf()).getMount();
File parent = f.getParentFile();
File dir = null;
while (parent != null && parent.canWrite() && parent.toString().startsWith(device)) {
dir = parent;
parent = parent.getParentFile();
}
if (dir == null) {
throw new IOException("not able to find the highest writable parent dir");
}
// move the file there
File badDir = new File(dir, "bad_files");
if (!badDir.mkdirs()) {
if (!badDir.isDirectory()) {
throw new IOException("Mkdirs failed to create " + badDir.toString());
}
}
String suffix = "." + rand.nextInt();
File badFile = new File(badDir, f.getName() + suffix);
LOG.warn("Moving bad file " + f + " to " + badFile);
// close it first
in.close();
// rename it
boolean b = f.renameTo(badFile);
if (!b) {
LOG.warn("Ignoring failure of renameTo");
}
// move checksum file too
File checkFile = ((RawLocalFileSystem) fs).pathToFile(getChecksumFile(p));
b = checkFile.renameTo(new File(badDir, checkFile.getName() + suffix));
if (!b) {
LOG.warn("Ignoring failure of renameTo");
}
} catch (IOException e) {
LOG.warn("Error moving bad file " + p + ": " + e);
}
return false;
public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException
long startTime = System.currentTimeMillis();
IOException ioe;
while (true) {
try {
return getProtocolProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout);
} catch (ConnectException se) {
// namenode has not been started
LOG.info("Server at " + addr + " not available yet, Zzzzz...");
ioe = se;
} catch (SocketTimeoutException te) {
// namenode is busy
LOG.info("Problem connecting to server: " + addr);
ioe = te;
} catch (NoRouteToHostException nrthe) {
// perhaps a VIP is failing over
LOG.info("No route to host for server: " + addr);
ioe = nrthe;
}
// check if timed out
if (System.currentTimeMillis() - timeout >= startTime) {
throw ioe;
}
// wait for retry
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
// IGNORE
}
}
this is very ugly, but needed to avoid breaking hdfs tests...
if a path has no authority, then the FileStatus from globStatus
will add the \"-fs\" authority into the path, so we need to sub
it back out to satisfy the tests
this is very ugly, but needed to avoid breaking hdfs tests...
if a path has no authority, then the FileStatus from globStatus
will add the \"-fs\" authority into the path, so we need to sub
it back out to satisfy the tests
FILE_PATH_CHANGED
expandAsGlob(String, Configuration)
public static PathData[] expandAsGlob(String pattern, Configuration conf) throws IOException
Path globPath = new Path(pattern);
FileSystem fs = globPath.getFileSystem(conf);
FileStatus[] stats = fs.globStatus(globPath);
PathData[] items = null;
if (stats == null) {
// not a glob & file not found, so add the path with a null stat
items = new PathData[] { new PathData(fs, pattern, null) };
} else if (// this is very ugly, but needed to avoid breaking hdfs tests...
// if a path has no authority, then the FileStatus from globStatus
// will add the "-fs" authority into the path, so we need to sub
// it back out to satisfy the tests
stats.length == 1 && stats[0].getPath().equals(fs.makeQualified(globPath))) {
// if the fq path is identical to the pattern passed, use the pattern
// to initialize the string value
items = new PathData[] { new PathData(fs, pattern, stats[0]) };
} else {
// convert stats to PathData
items = new PathData[stats.length];
int i = 0;
for (FileStatus stat : stats) {
items[i++] = new PathData(fs, stat);
}
}
return items;
Ignored the mbean itself was not found, which should never happen because we
just accessed it (perhaps something unregistered in-between) but if this
happens just don't output the attribute.
Ignored the mbean itself was not found, which should never happen because we
just accessed it (perhaps something unregistered in-between) but if this
happens just don't output the attribute.
if (!attr.isReadable()) {
return;
}
String attName = attr.getName();
if ("modelerType".equals(attName)) {
return;
}
if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 || attName.indexOf(" ") >= 0) {
return;
}
Object value = null;
try {
value = mBeanServer.getAttribute(oname, attName);
} catch (AttributeNotFoundException e) {
// Ignored the attribute was not found, which should never happen because the bean
// just told us that it has this attribute, but if this happens just don't output
// the attribute.
return;
} catch (MBeanException e) {
// The code inside the attribute getter threw an exception so log it, and
// skip outputting the attribute
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (RuntimeException e) {
// For some reason even with an MBeanException available to them Runtime exceptions
// can still find their way through, so treat them the same as MBeanException
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (ReflectionException e) {
// This happens when the code inside the JMX bean (setter?? from the java docs)
// threw an exception, so log it and skip outputting the attribute
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (InstanceNotFoundException e) {
// Ignored the mbean itself was not found, which should never happen because we
// just accessed it (perhaps something unregistered in-between) but if this
// happens just don't output the attribute.
return;
}
writeAttribute(jg, attName, value);
public FileMetadata retrieveMetadata(String key) throws IOException
try {
S3Object object = s3Service.getObjectDetails(bucket, key);
return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime());
} catch (S3ServiceException e) {
// Following is brittle. Is there a better way?
if (e.getMessage().contains("ResponseCode=404")) {
return null;
}
handleServiceException(e);
// never returned - keep compiler happy
return null;
}
walk through the list, searching. Not the most efficient way, but this
in intended to be used rarely, so we keep it simple.
As an optimization, we can keep a hashmap of record name to its RTI, for later.
walk through the list, searching. Not the most efficient way, but this
in intended to be used rarely, so we keep it simple.
As an optimization, we can keep a hashmap of record name to its RTI, for later.
FILE_PATH_CHANGED
findStruct(String)
// walk through the list, searching. Not the most efficient way, but this
// in intended to be used rarely, so we keep it simple.
// As an optimization, we can keep a hashmap of record name to its RTI, for later.
for (FieldTypeInfo ti : typeInfos) {
if ((0 == ti.getFieldID().compareTo(name)) && (ti.getTypeID().getTypeVal() == RIOType.STRUCT)) {
return (StructTypeID) ti.getTypeID();
}
}
return null;
Now another magic 48-bit number, 0x177245385090, to indicate the end
of the last block. (sqrt(pi), if you want to know. I did want to use
e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me
to feel statistically comfortable. Call me paranoid.)
Now another magic 48-bit number, 0x177245385090, to indicate the end
of the last block. (sqrt(pi), if you want to know. I did want to use
e, but it contains too much repetition -- 27 18 28 18 28 46 -- for me
to feel statistically comfortable. Call me paranoid.)
FILE_PATH_CHANGED
hbMakeCodeLengths(char[], int[], int, int)
protected static void hbMakeCodeLengths(char[] len, int[] freq, int alphaSize, int maxLen)
/*
* Nodes and heap entries run from 1. Entry 0 for both the heap and
* nodes is a sentinel.
*/
final int[] heap = new int[MAX_ALPHA_SIZE * 2];
final int[] weight = new int[MAX_ALPHA_SIZE * 2];
final int[] parent = new int[MAX_ALPHA_SIZE * 2];
for (int i = alphaSize; --i >= 0; ) {
weight[i + 1] = (freq[i] == 0 ? 1 : freq[i]) << 8;
}
for (boolean tooLong = true; tooLong; ) {
tooLong = false;
int nNodes = alphaSize;
int nHeap = 0;
heap[0] = 0;
weight[0] = 0;
parent[0] = -2;
for (int i = 1; i <= alphaSize; i++) {
parent[i] = -1;
nHeap++;
heap[nHeap] = i;
int zz = nHeap;
int tmp = heap[zz];
while (weight[tmp] < weight[heap[zz >> 1]]) {
heap[zz] = heap[zz >> 1];
zz >>= 1;
}
heap[zz] = tmp;
}
// assert (nHeap < (MAX_ALPHA_SIZE + 2)) : nHeap;
while (nHeap > 1) {
int n1 = heap[1];
heap[1] = heap[nHeap];
nHeap--;
int yy = 0;
int zz = 1;
int tmp = heap[1];
while (true) {
yy = zz << 1;
if (yy > nHeap) {
break;
}
if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) {
yy++;
}
if (weight[tmp] < weight[heap[yy]]) {
break;
}
heap[zz] = heap[yy];
zz = yy;
}
heap[zz] = tmp;
int n2 = heap[1];
heap[1] = heap[nHeap];
nHeap--;
yy = 0;
zz = 1;
tmp = heap[1];
while (true) {
yy = zz << 1;
if (yy > nHeap) {
break;
}
if ((yy < nHeap) && (weight[heap[yy + 1]] < weight[heap[yy]])) {
yy++;
}
if (weight[tmp] < weight[heap[yy]]) {
break;
}
heap[zz] = heap[yy];
zz = yy;
}
heap[zz] = tmp;
nNodes++;
parent[n1] = parent[n2] = nNodes;
final int weight_n1 = weight[n1];
final int weight_n2 = weight[n2];
weight[nNodes] = (((weight_n1 & 0xffffff00) + (weight_n2 & 0xffffff00)) | (1 + (((weight_n1 & 0x000000ff) > (weight_n2 & 0x000000ff)) ? (weight_n1 & 0x000000ff) : (weight_n2 & 0x000000ff))));
parent[nNodes] = -1;
nHeap++;
heap[nHeap] = nNodes;
tmp = 0;
zz = nHeap;
tmp = heap[zz];
final int weight_tmp = weight[tmp];
while (weight_tmp < weight[heap[zz >> 1]]) {
heap[zz] = heap[zz >> 1];
zz >>= 1;
}
heap[zz] = tmp;
}
// assert (nNodes < (MAX_ALPHA_SIZE * 2)) : nNodes;
for (int i = 1; i <= alphaSize; i++) {
int j = 0;
int k = i;
for (int parent_k; (parent_k = parent[k]) >= 0; ) {
k = parent_k;
j++;
}
len[i - 1] = (char) j;
if (j > maxLen) {
tooLong = true;
}
}
if (tooLong) {
for (int i = 1; i < alphaSize; i++) {
int j = weight[i] >> 8;
j = 1 + (j >> 1);
weight[i] = j << 8;
}
}
}
public FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum, boolean createParent) throws IOException
checkPath(f);
// Default impl assumes that permissions do not matter
// calling the regular create is good enough.
// FSs that implement permissions should override this.
if (!createParent) {
// parent must exist.
// since this.create makes parent dirs automatically
// we must throw exception if parent does not exist.
final FileStatus stat = getFileStatus(f.getParent());
if (stat == null) {
throw new FileNotFoundException("Missing parent:" + f);
}
if (!stat.isDirectory()) {
throw new ParentNotDirectoryException("parent is not a dir:" + f);
}
// parent does exist - go ahead with create of file.
}
return fsImpl.primitiveCreate(f, absolutePermission, flag, bufferSize, replication, blockSize, progress, bytesPerChecksum);
long length = ftpFile.getSize();
boolean isDir = ftpFile.isDirectory();
int blockReplication = 1;
// Using default block size since there is no way in FTP client to know of
// block sizes on server. The assumption could be less than ideal.
long blockSize = DEFAULT_BLOCK_SIZE;
long modTime = ftpFile.getTimestamp().getTimeInMillis();
long accessTime = 0;
FsPermission permission = getPermissions(ftpFile);
String user = ftpFile.getUser();
String group = ftpFile.getGroup();
Path filePath = new Path(parentPath, ftpFile.getName());
return new FileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, permission, user, group, filePath.makeQualified(this));
Check that every permission has its sticky bit represented correctly
Check that every permission has its sticky bit represented correctly
FILE_PATH_CHANGED
testStickyBitToString()
public void testStickyBitToString()
// Check that every permission has its sticky bit represented correctly
for (boolean sb : new boolean[] { false, true }) {
for (FsAction u : FsAction.values()) {
for (FsAction g : FsAction.values()) {
for (FsAction o : FsAction.values()) {
FsPermission f = new FsPermission(u, g, o, sb);
if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('t', f.toString().charAt(8));
else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE))
assertEquals('T', f.toString().charAt(8));
else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('x', f.toString().charAt(8));
else
assertEquals('-', f.toString().charAt(8));
}
}
}
}
TODO: if the user wants the trash to be used but there is any
problem (ie. creating the trash dir, moving the item to be deleted,
etc), then the path will just be deleted because moveToTrash returns
false and it falls thru to fs.delete. this doesn't seem right
TODO: if the user wants the trash to be used but there is any
problem (ie. creating the trash dir, moving the item to be deleted,
etc), then the path will just be deleted because moveToTrash returns
false and it falls thru to fs.delete. this doesn't seem right
FILE_PATH_CHANGED
registerCommands(CommandFactory)
public static void registerCommands(CommandFactory factory)
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
FILE_PATH_CHANGED
testOldFormat()
public void testOldFormat() throws IOException
// Make sure we still correctly write the old format if desired.
// Write the data array with old ObjectWritable API
// which will set allowCompactArrays false.
ObjectWritable.writeObject(out, i, i.getClass(), null);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
@SuppressWarnings("deprecation")
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not labelled as an array of int", i.getClass().getName(), className);
int length = in.readInt();
assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not expected length", i.length, length);
int[] readValue = new int[length];
try {
for (int i = 0; i < length; i++) {
readValue[i] = (int) ((Integer) ObjectWritable.readObject(in, null));
}
} catch (Exception e) {
fail("The int[] written by ObjectWritable as a non-compact array " + "was corrupted. Failed to correctly read int[] of length " + length + ". Got exception:\n" + StringUtils.stringifyException(e));
}
assertTrue("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.", Arrays.equals(i, readValue));
Read the APW object as written by ObjectWritable, but
\"going around\" ObjectWritable
Read the APW object as written by ObjectWritable, but
\"going around\" ObjectWritable
FILE_PATH_CHANGED
testObjectLabeling()
public void testObjectLabeling() throws IOException
// Do a few tricky experiments to make sure things are being written
// the way we expect
// Write the data array with ObjectWritable
// which will indirectly write it using APW.Internal
ObjectWritable.writeObject(out, i, i.getClass(), null, true);
// Write the corresponding APW directly with ObjectWritable
ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i);
ObjectWritable.writeObject(out, apw, apw.getClass(), null, true);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className);
ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal();
apwi.readFields(in);
assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get())));
// Read the APW object as written by ObjectWritable, but
// "going around" ObjectWritable
String declaredClassName = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName);
className = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className);
ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable();
apw2.readFields(in);
assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
FILE_PATH_CHANGED
testObjectLabeling()
public void testObjectLabeling() throws IOException
// Do a few tricky experiments to make sure things are being written
// the way we expect
// Write the data array with ObjectWritable
// which will indirectly write it using APW.Internal
ObjectWritable.writeObject(out, i, i.getClass(), null, true);
// Write the corresponding APW directly with ObjectWritable
ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i);
ObjectWritable.writeObject(out, apw, apw.getClass(), null, true);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className);
ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal();
apwi.readFields(in);
assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get())));
// Read the APW object as written by ObjectWritable, but
// "going around" ObjectWritable
String declaredClassName = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName);
className = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className);
ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable();
apw2.readFields(in);
assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
NOTE: this is the one place we do NOT want to save the number
of bytes sent (nRemaining here) into lastBytesSent: since we
are resending what we've already sent before, offset is nonzero
in general (only way it could be zero is if it already equals
nRemaining), which would then screw up the offset calculation
_next_ time around. IOW, getRemaining() is in terms of the
original, zero-offset bufferload, so lastBytesSent must be as
well. Cheesy ASCII art:
<------------ m, lastBytesSent ----------->
+===============================================+
buffer: |1111111111|22222222222222222|333333333333| |
+===============================================+
#1: <-- off -->|<-------- nRemaining --------->
#2: <----------- off ----------->|<-- nRem. -->
NOTE: this is the one place we do NOT want to save the number
of bytes sent (nRemaining here) into lastBytesSent: since we
are resending what we've already sent before, offset is nonzero
in general (only way it could be zero is if it already equals
nRemaining), which would then screw up the offset calculation
_next_ time around. IOW, getRemaining() is in terms of the
original, zero-offset bufferload, so lastBytesSent must be as
well. Cheesy ASCII art:
<------------ m, lastBytesSent ----------->
+===============================================+
buffer: |1111111111|22222222222222222|333333333333| |
+===============================================+
#1: <-- off -->|<-------- nRemaining --------->
#2: <----------- off ----------->|<-- nRem. -->
Default impl is to assume that permissions do not matter and hence
calling the regular mkdirs is good enough.
FSs that implement permissions should override this.
Default impl is to assume that permissions do not matter and hence
calling the regular mkdirs is good enough.
FSs that implement permissions should override this.
Default impl assumes that permissions do not matter and
nor does the bytesPerChecksum hence
calling the regular create is good enough.
FSs that implement permissions should override this.
Default impl assumes that permissions do not matter and
nor does the bytesPerChecksum hence
calling the regular create is good enough.
FSs that implement permissions should override this.
public static String[] split(String str, char escapeChar, char separator)
if (str == null) {
return null;
}
ArrayList strList = new ArrayList();
StringBuilder split = new StringBuilder();
int index = 0;
while ((index = findNext(str, separator, escapeChar, index, split)) >= 0) {
// move over the separator for next search
++index;
strList.add(split.toString());
// reset the buffer
split.setLength(0);
}
strList.add(split.toString());
// remove trailing empty split(s)
// last split
int last = strList.size();
while (--last >= 0 && "".equals(strList.get(last))) {
strList.remove(last);
}
return strList.toArray(new String[strList.size()]);
FIXME? Inflater docs say: 'it is also necessary to provide an extra
\"dummy\" byte as input. This is required by the ZLIB native
library in order to support certain optimizations.' However,
this does not appear to be true, and in any case, it's not
entirely clear where the byte should go or what its value
should be. Perhaps it suffices to have some deflated bytes
in the first buffer load? (But how else would one do it?)
FIXME? Inflater docs say: 'it is also necessary to provide an extra
\"dummy\" byte as input. This is required by the ZLIB native
library in order to support certain optimizations.' However,
this does not appear to be true, and in any case, it's not
entirely clear where the byte should go or what its value
should be. Perhaps it suffices to have some deflated bytes
in the first buffer load? (But how else would one do it?)
FILE_PATH_CHANGED
needsInput()
public synchronized boolean needsInput()
if (state == GzipStateLabel.DEFLATE_STREAM) {
// most common case
return inflater.needsInput();
}
// see userBufLen comment at top of decompress(); currently no need to
// verify userBufLen <= 0
return (state != GzipStateLabel.FINISHED);
Check if zlib consumed all input buffer
set keepUncompressedBuf properly
zlib consumed all input buffer
Check if zlib consumed all input buffer
set keepUncompressedBuf properly
zlib consumed all input buffer
FILE_PATH_CHANGED
compress(byte[], int, int)
public synchronized int compress(byte[] b, int off, int len) throws IOException
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
int n = 0;
// Check if there is compressed data
n = compressedDirectBuf.remaining();
if (n > 0) {
n = Math.min(n, len);
((ByteBuffer) compressedDirectBuf).get(b, off, n);
return n;
}
// Re-initialize the zlib's output direct buffer
compressedDirectBuf.rewind();
compressedDirectBuf.limit(directBufferSize);
// Compress data
n = deflateBytesDirect();
compressedDirectBuf.limit(n);
// Check if zlib consumed all input buffer
// set keepUncompressedBuf properly
if (uncompressedDirectBufLen <= 0) {
// zlib consumed all input buffer
keepUncompressedBuf = false;
uncompressedDirectBuf.clear();
uncompressedDirectBufOff = 0;
uncompressedDirectBufLen = 0;
} else {
// zlib did not consume all input buffer
keepUncompressedBuf = true;
}
// Get atmost 'len' bytes
n = Math.min(n, len);
((ByteBuffer) compressedDirectBuf).get(b, off, n);
return n;
TO DO: - more efficient to not split the path, but simply compare
TO DO: - more efficient to not split the path, but simply compare
FILE_PATH_CHANGED
resolve(String, boolean)
// TO DO: - more efficient to not split the path, but simply compare
String[] path = breakIntoPathComponents(p);
if (path.length <= 1) {
// special case for when path is "/"
ResolveResult res = new ResolveResult(ResultKind.isInternalDir, root.InodeDirFs, root.fullPath, SlashPath);
return res;
}
INodeDir curInode = root;
int i;
// ignore first slash
for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) {
INode nextInode = curInode.resolveInternal(path[i]);
if (nextInode == null) {
StringBuilder failedAt = new StringBuilder(path[0]);
for (int j = 1; j <= i; ++j) {
failedAt.append('/').append(path[j]);
}
throw (new FileNotFoundException(failedAt.toString()));
}
if (nextInode instanceof INodeLink) {
final INodeLink link = (INodeLink) nextInode;
final Path remainingPath;
if (i >= path.length - 1) {
remainingPath = SlashPath;
} else {
StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]);
for (int j = i + 2; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult res = new ResolveResult(ResultKind.isExternalDir, link.targetFileSystem, nextInode.fullPath, remainingPath);
return res;
} else if (nextInode instanceof INodeDir) {
curInode = (INodeDir) nextInode;
}
}
// We have resolved to an internal dir in mount table.
Path remainingPath;
if (resolveLastComponent) {
remainingPath = SlashPath;
} else {
// note we have taken care of when path is "/" above
// for internal dirs rem-path does not start with / since the lookup
// that follows will do a children.get(remaningPath) and will have to
// strip-out the initial /
StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
for (int j = i + 1; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult res = new ResolveResult(ResultKind.isInternalDir, curInode.InodeDirFs, curInode.fullPath, remainingPath);
return res;
ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException
Ideally we should wait after transferTo returns 0. But because of
a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988),
which throws an exception instead of returning 0, we wait for the
channel to be writable before writing to it. If you ever see
IOException with message \"Resource temporarily unavailable\"
thrown here, please let us know.
Once we move to JAVA SE 7, wait should be moved to correct place.
Ideally we should wait after transferTo returns 0. But because of
a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988),
which throws an exception instead of returning 0, we wait for the
channel to be writable before writing to it. If you ever see
IOException with message \"Resource temporarily unavailable\"
thrown here, please let us know.
Once we move to JAVA SE 7, wait should be moved to correct place.
FILE_PATH_CHANGED
transferToFully(FileChannel, long, int)
public void transferToFully(FileChannel fileCh, long position, int count) throws IOException
while (count > 0) {
/*
* Ideally we should wait after transferTo returns 0. But because of
* a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988),
* which throws an exception instead of returning 0, we wait for the
* channel to be writable before writing to it. If you ever see
* IOException with message "Resource temporarily unavailable"
* thrown here, please let us know.
*
* Once we move to JAVA SE 7, wait should be moved to correct place.
*/
waitForWritable();
int nTransfered = (int) fileCh.transferTo(position, count, getChannel());
if (nTransfered == 0) {
// check if end of file is reached.
if (position >= fileCh.size()) {
throw new EOFException("EOF Reached. file size is " + fileCh.size() + " and " + count + " more bytes left to be " + "transfered.");
}
// otherwise assume the socket is full.
// waitForWritable(); // see comment above.
} else if (nTransfered < 0) {
throw new IOException("Unexpected return of " + nTransfered + " from transferTo()");
} else {
position += nTransfered;
count -= nTransfered;
}
}
TODO: DFSAdmin subclasses FsShell so need to protect the command
registration. This class should morph into a base class for
commands, and then this method can be abstract
TODO: DFSAdmin subclasses FsShell so need to protect the command
registration. This class should morph into a base class for
commands, and then this method can be abstract
// Do the authorization
if (HttpServer.hasAdministratorAccess(getServletContext(), request, response)) {
// Authorization is done. Just call super.
super.doGet(request, response);
}
TODO: sample the generated key/value records, and put the numbers below
TODO: sample the generated key/value records, and put the numbers below
FILE_PATH_CHANGED
setUp()
public void setUp() throws IOException
skip = !(Algorithm.LZO.isSupported());
if (skip) {
System.out.println("Skipped");
}
// TODO: sample the generated key/value records, and put the numbers below
init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo", 2605, 2558);
if (!skip)
super.setUp();
if (item.stat.isDirectory()) {
// TODO: handle this
throw new PathIsDirectoryException(item.toString());
}
if (item.stat.getLen() != 0) {
throw new PathIOException(item.toString(), "Not a zero-length file");
}
touchz(item);
Read characters into a portion of an array, reading from the underlying
stream at most once if necessary.
Read characters into a portion of an array, reading from the underlying
stream at most once if necessary.
FILE_PATH_CHANGED
read1(byte[], int, int)
private int read1(byte[] b, int off, int len) throws IOException
int avail = count - pos;
if (avail <= 0) {
if (len >= maxChunkSize) {
// read a chunk to user buffer directly; avoid one copy
int nread = readChecksumChunk(b, off, len);
return nread;
} else {
// read a chunk into the local buffer
fill();
if (count <= 0) {
return -1;
} else {
avail = count;
}
}
}
// copy content of the local buffer to the user buffer
int cnt = (avail < len) ? avail : len;
System.arraycopy(buf, pos, b, off, cnt);
pos += cnt;
return cnt;
TODO: DFSAdmin subclasses FsShell so need to protect the command
registration. This class should morph into a base class for
commands, and then this method can be abstract
TODO: This isn't the best place, but this class is being abused with
subclasses which of course override this method. There really needs
to be a better base class for all commands
Ignored the mbean itself was not found, which should never happen because we
just accessed it (perhaps something unregistered in-between) but if this
happens just don't output the attribute.
if (!attr.isReadable()) {
return;
}
String attName = attr.getName();
if ("modelerType".equals(attName)) {
return;
}
if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 || attName.indexOf(" ") >= 0) {
return;
}
Object value = null;
try {
value = mBeanServer.getAttribute(oname, attName);
} catch (AttributeNotFoundException e) {
// Ignored the attribute was not found, which should never happen because the bean
// just told us that it has this attribute, but if this happens just don't output
// the attribute.
return;
} catch (MBeanException e) {
// The code inside the attribute getter threw an exception so log it, and
// skip outputting the attribute
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (RuntimeException e) {
// For some reason even with an MBeanException available to them Runtime exceptions
// can still find their way through, so treat them the same as MBeanException
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (ReflectionException e) {
// This happens when the code inside the JMX bean (setter?? from the java docs)
// threw an exception, so log it and skip outputting the attribute
LOG.error("getting attribute " + attName + " of " + oname + " threw an exception", e);
return;
} catch (InstanceNotFoundException e) {
// Ignored the mbean itself was not found, which should never happen because we
// just accessed it (perhaps something unregistered in-between) but if this
// happens just don't output the attribute.
return;
}
writeAttribute(jg, attName, value);
Doing a second call with faults disabled should return fine --
ie the internal state of the client or server should not be broken
by the failed call
SATD_ADDED
doErrorTest(Class, Class, Class, Class)
private void doErrorTest(Class clientParamClass, Class serverParamClass, Class serverResponseClass, Class clientResponseClass) throws Exception
// start server
Server server = new TestServer(1, false, serverParamClass, serverResponseClass);
InetSocketAddress addr = NetUtils.getConnectAddress(server);
server.start();
// start client
WRITABLE_FAULTS_ENABLED = true;
Client client = new Client(clientResponseClass, conf);
try {
LongWritable param = clientParamClass.newInstance();
try {
client.call(param, addr, null, null, 0, conf);
fail("Expected an exception to have been thrown");
} catch (Throwable t) {
assertExceptionContains(t, "Injected fault");
}
// Doing a second call with faults disabled should return fine --
// ie the internal state of the client or server should not be broken
// by the failed call
WRITABLE_FAULTS_ENABLED = false;
client.call(param, addr, null, null, 0, conf);
} finally {
server.stop();
}
TODO: if the user wants the trash to be used but there is any
problem (ie. creating the trash dir, moving the item to be deleted,
etc), then the path will just be deleted because moveToTrash returns
false and it falls thru to fs.delete. this doesn't seem right
SATD_ADDED
registerCommands(CommandFactory)
public static void registerCommands(CommandFactory factory)
if (item.stat.isDirectory()) {
// TODO: handle this
throw new PathIsDirectoryException(item.toString());
}
if (item.stat.getLen() != 0) {
throw new PathIOException(item.toString(), "Not a zero-length file");
}
touchz(item);
TODO: this is a quick workaround to accelerate the integration of
redesigned commands. this will be removed this once all commands are
converted. this change will avoid having to change the hdfs tests
every time a command is converted to use path-based exceptions
SATD_ADDED
getFS()
protected FileSystem getFS() throws IOException
if (fs == null)
fs = FileSystem.get(getConf());
return fs;
public boolean mkdirs(Path dir, FsPermission permission) throws AccessControlException, FileAlreadyExistsException
if (theInternalDir.isRoot & dir == null) {
throw new FileAlreadyExistsException("/ already exits");
}
// Note dir starts with /
if (theInternalDir.children.containsKey(dir.toString().substring(1))) {
// this is the stupid semantics of FileSystem
return true;
}
throw readOnlyMountTable("mkdirs", dir);
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFileSystemFileStatus that
works around.
SATD_ADDED
readOnlyMountTable(String, String)
static AccessControlException readOnlyMountTable(final String operation, final String p)
return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
TO DO: - more efficient to not split the path, but simply compare
SATD_ADDED
resolve(String, boolean)
// TO DO: - more efficient to not split the path, but simply compare
String[] path = breakIntoPathComponents(p);
if (path.length <= 1) {
// special case for when path is "/"
ResolveResult res = new ResolveResult(ResultKind.isInternalDir, root.InodeDirFs, root.fullPath, SlashPath);
return res;
}
INodeDir curInode = root;
int i;
// ignore first slash
for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) {
INode nextInode = curInode.resolveInternal(path[i]);
if (nextInode == null) {
StringBuilder failedAt = new StringBuilder(path[0]);
for (int j = 1; j <= i; ++j) {
failedAt.append('/').append(path[j]);
}
throw (new FileNotFoundException(failedAt.toString()));
}
if (nextInode instanceof INodeLink) {
final INodeLink link = (INodeLink) nextInode;
final Path remainingPath;
if (i >= path.length - 1) {
remainingPath = SlashPath;
} else {
StringBuilder remainingPathStr = new StringBuilder("/" + path[i + 1]);
for (int j = i + 2; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult res = new ResolveResult(ResultKind.isExternalDir, link.targetFileSystem, nextInode.fullPath, remainingPath);
return res;
} else if (nextInode instanceof INodeDir) {
curInode = (INodeDir) nextInode;
}
}
// We have resolved to an internal dir in mount table.
Path remainingPath;
if (resolveLastComponent) {
remainingPath = SlashPath;
} else {
// note we have taken care of when path is "/" above
// for internal dirs rem-path does not start with / since the lookup
// that follows will do a children.get(remaningPath) and will have to
// strip-out the initial /
StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
for (int j = i + 1; j < path.length; ++j) {
remainingPathStr.append('/').append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
final ResolveResult res = new ResolveResult(ResultKind.isInternalDir, curInode.InodeDirFs, curInode.fullPath, remainingPath);
return res;
ResolveResult resolve(final String p, final boolean resolveLastComponent) throws FileNotFoundException
The implementors of RawLocalFileSystem were trying to be very smart.
They implement FileStatus#getOwener lazily -- the object
returned is really a RawLocalFileSystem that expect the
FileStatus#getPath to be unchanged so that it can get owner when needed.
Hence we need to interpose a new ViewFsFileStatus that works around.
SATD_ADDED
readOnlyMountTable(String, String)
static AccessControlException readOnlyMountTable(final String operation, final String p)
return new AccessControlException("InternalDir of ViewFileSystem is readonly; operation=" + operation + "Path=" + p);
this is very ugly, but needed to avoid breaking hdfs tests...
if a path has no authority, then the FileStatus from globStatus
will add the \"-fs\" authority into the path, so we need to sub
it back out to satisfy the tests
SATD_ADDED
expandAsGlob(String, Configuration)
public static PathData[] expandAsGlob(String pattern, Configuration conf) throws IOException
Path globPath = new Path(pattern);
FileSystem fs = globPath.getFileSystem(conf);
FileStatus[] stats = fs.globStatus(globPath);
PathData[] items = null;
if (stats == null) {
// not a glob & file not found, so add the path with a null stat
items = new PathData[] { new PathData(fs, pattern, null) };
} else if (// this is very ugly, but needed to avoid breaking hdfs tests...
// if a path has no authority, then the FileStatus from globStatus
// will add the "-fs" authority into the path, so we need to sub
// it back out to satisfy the tests
stats.length == 1 && stats[0].getPath().equals(fs.makeQualified(globPath))) {
// if the fq path is identical to the pattern passed, use the pattern
// to initialize the string value
items = new PathData[] { new PathData(fs, pattern, stats[0]) };
} else {
// convert stats to PathData
items = new PathData[stats.length];
int i = 0;
for (FileStatus stat : stats) {
items[i++] = new PathData(fs, stat);
}
}
return items;
TODO: This isn't the best place, but this class is being abused with
subclasses which of course override this method. There really needs
to be a better base class for all commands
TODO: This isn't the best place, but this class is being abused with
subclasses which of course override this method. There really needs
to be a better base class for all commands
CLASS_OR_METHOD_CHANGED
getFS()
protected FileSystem getFS() throws IOException
if (fs == null)
fs = FileSystem.get(getConf());
return fs;
Keep the structure similar to ChecksumFileSystem.copyToLocal().
Ideal these two should just invoke FileUtil.copy() and not repeat
recursion here. Of course, copy() should support two more options :
copyCrc and useTmpFile (may be useTmpFile need not be an option).
Keep the structure similar to ChecksumFileSystem.copyToLocal().
Ideal these two should just invoke FileUtil.copy() and not repeat
recursion here. Of course, copy() should support two more options :
copyCrc and useTmpFile (may be useTmpFile need not be an option).
CLASS_OR_METHOD_CHANGED
getFS()
protected FileSystem getFS() throws IOException
if (fs == null)
fs = FileSystem.get(getConf());
return fs;
TODO: This isn't the best place, but this class is being abused with
subclasses which of course override this method. There really needs
to be a better base class for all commands
SATD_ADDED
init()
protected void init() throws IOException
getConf().setQuietMode(true);
if (this.fs == null) {
this.fs = FileSystem.get(getConf());
}
if (this.trash == null) {
this.trash = new Trash(getConf());
}
this is an unexpected condition, so dump the whole exception since
it's probably a nasty internal error where the backtrace would be
useful
SATD_ADDED
displayError(Exception)
public void displayError(Exception e)
// build up a list of exceptions that occurred
exceptions.add(e);
String errorMessage = e.getLocalizedMessage();
if (errorMessage == null) {
// this is an unexpected condition, so dump the whole exception since
// it's probably a nasty internal error where the backtrace would be
// useful
errorMessage = StringUtils.stringifyException(e);
LOG.debug(errorMessage);
} else {
errorMessage = errorMessage.split("\n", 2)[0];
}
displayError(errorMessage);
glob failed to match
TODO: this should be more posix-like: ex. \"No such file or directory\"
SATD_ADDED
expandGlob(String)
protected List expandGlob(String pattern) throws IOException
Path path = new Path(pattern);
FileSystem fs = path.getFileSystem(getConf());
FileStatus[] stats = fs.globStatus(path);
if (stats != null && stats.length == 0) {
// glob failed to match
// TODO: this should be more posix-like: ex. "No such file or directory"
throw new FileNotFoundException("Can not find listing for " + pattern);
}
List items = new LinkedList();
if (stats == null) {
// not a glob & file not found, so null stat block
items.add(new PathData(fs, path, null));
} else {
// convert all the stats to PathData objs
for (FileStatus stat : stats) {
items.add(new PathData(fs, stat));
}
}
return items;
this is an unexpected condition, so dump the whole exception since
it's probably a nasty internal error where the backtrace would be
useful
SATD_ADDED
displayError(Exception)
public void displayError(Exception e)
// build up a list of exceptions that occurred
exceptions.add(e);
String errorMessage = e.getLocalizedMessage();
if (errorMessage == null) {
// this is an unexpected condition, so dump the whole exception since
// it's probably a nasty internal error where the backtrace would be
// useful
errorMessage = StringUtils.stringifyException(e);
LOG.debug(errorMessage);
} else {
errorMessage = errorMessage.split("\n", 2)[0];
}
displayError(errorMessage);
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
SATD_ADDED
testOldFormat()
public void testOldFormat() throws IOException
// Make sure we still correctly write the old format if desired.
// Write the data array with old ObjectWritable API
// which will set allowCompactArrays false.
ObjectWritable.writeObject(out, i, i.getClass(), null);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
@SuppressWarnings("deprecation")
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not labelled as an array of int", i.getClass().getName(), className);
int length = in.readInt();
assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not expected length", i.length, length);
int[] readValue = new int[length];
try {
for (int i = 0; i < length; i++) {
readValue[i] = (int) ((Integer) ObjectWritable.readObject(in, null));
}
} catch (Exception e) {
fail("The int[] written by ObjectWritable as a non-compact array " + "was corrupted. Failed to correctly read int[] of length " + length + ". Got exception:\n" + StringUtils.stringifyException(e));
}
assertTrue("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.", Arrays.equals(i, readValue));
Read the APW object as written by ObjectWritable, but
\"going around\" ObjectWritable
SATD_ADDED
testObjectLabeling()
public void testObjectLabeling() throws IOException
// Do a few tricky experiments to make sure things are being written
// the way we expect
// Write the data array with ObjectWritable
// which will indirectly write it using APW.Internal
ObjectWritable.writeObject(out, i, i.getClass(), null, true);
// Write the corresponding APW directly with ObjectWritable
ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i);
ObjectWritable.writeObject(out, apw, apw.getClass(), null, true);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className);
ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal();
apwi.readFields(in);
assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get())));
// Read the APW object as written by ObjectWritable, but
// "going around" ObjectWritable
String declaredClassName = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName);
className = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className);
ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable();
apw2.readFields(in);
assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
Read the int[] object as written by ObjectWritable, but
\"going around\" ObjectWritable
SATD_ADDED
testObjectLabeling()
public void testObjectLabeling() throws IOException
// Do a few tricky experiments to make sure things are being written
// the way we expect
// Write the data array with ObjectWritable
// which will indirectly write it using APW.Internal
ObjectWritable.writeObject(out, i, i.getClass(), null, true);
// Write the corresponding APW directly with ObjectWritable
ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable(i);
ObjectWritable.writeObject(out, apw, apw.getClass(), null, true);
// Get ready to read it back
in.reset(out.getData(), out.getLength());
// Read the int[] object as written by ObjectWritable, but
// "going around" ObjectWritable
String className = UTF8.readString(in);
assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal", ArrayPrimitiveWritable.Internal.class.getName(), className);
ArrayPrimitiveWritable.Internal apwi = new ArrayPrimitiveWritable.Internal();
apwi.readFields(in);
assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted", int.class, apw.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted", Arrays.equals(i, (int[]) (apwi.get())));
// Read the APW object as written by ObjectWritable, but
// "going around" ObjectWritable
String declaredClassName = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), declaredClassName);
className = UTF8.readString(in);
assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable", ArrayPrimitiveWritable.class.getName(), className);
ArrayPrimitiveWritable apw2 = new ArrayPrimitiveWritable();
apw2.readFields(in);
assertEquals("The ArrayPrimitiveWritable component type was corrupted", int.class, apw2.getComponentType());
assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted", Arrays.equals(i, (int[]) (apw2.get())));
public static ProtocolProxy waitForProtocolProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException
long startTime = System.currentTimeMillis();
IOException ioe;
while (true) {
try {
return getProtocolProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout);
} catch (ConnectException se) {
// namenode has not been started
LOG.info("Server at " + addr + " not available yet, Zzzzz...");
ioe = se;
} catch (SocketTimeoutException te) {
// namenode is busy
LOG.info("Problem connecting to server: " + addr);
ioe = te;
} catch (NoRouteToHostException nrthe) {
// perhaps a VIP is failing over
LOG.info("No route to host for server: " + addr);
ioe = nrthe;
}
// check if timed out
if (System.currentTimeMillis() - timeout >= startTime) {
throw ioe;
}
// wait for retry
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
// IGNORE
}
}
Check if zlib consumed all input buffer
set keepUncompressedBuf properly
zlib consumed all input buffer
SATD_ADDED
compress(byte[], int, int)
public synchronized int compress(byte[] b, int off, int len) throws IOException
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
int n = 0;
// Check if there is compressed data
n = compressedDirectBuf.remaining();
if (n > 0) {
n = Math.min(n, len);
((ByteBuffer) compressedDirectBuf).get(b, off, n);
return n;
}
// Re-initialize the zlib's output direct buffer
compressedDirectBuf.rewind();
compressedDirectBuf.limit(directBufferSize);
// Compress data
n = deflateBytesDirect();
compressedDirectBuf.limit(n);
// Check if zlib consumed all input buffer
// set keepUncompressedBuf properly
if (uncompressedDirectBufLen <= 0) {
// zlib consumed all input buffer
keepUncompressedBuf = false;
uncompressedDirectBuf.clear();
uncompressedDirectBufOff = 0;
uncompressedDirectBufLen = 0;
} else {
// zlib did not consume all input buffer
keepUncompressedBuf = true;
}
// Get atmost 'len' bytes
n = Math.min(n, len);
((ByteBuffer) compressedDirectBuf).get(b, off, n);
return n;
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
if (name == null) {
LOG.warn("Metric was emitted with no name.");
return;
} else if (value == null) {
LOG.warn("Metric name " + name + " was emitted with a null value.");
return;
} else if (type == null) {
LOG.warn("Metric name " + name + ", value " + value + " has no type.");
return;
}
LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName);
String units = getUnits(name);
if (units == null) {
LOG.warn("Metric name " + name + ", value " + value + " had 'null' units");
units = "";
}
int slope = getSlope(name);
int tmax = getTmax(name);
int dmax = getDmax(name);
offset = 0;
String groupName = name.substring(0, name.lastIndexOf("."));
// The following XDR recipe was done through a careful reading of
// gm_protocol.x in Ganglia 3.1 and carefully examining the output of
// the gmetric utility with strace.
// First we send out a metadata message
// metric_id = metadata_msg
xdr_int(128);
// hostname
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// metric type
xdr_string(type);
// metric name
xdr_string(name);
// units
xdr_string(units);
// slope
xdr_int(slope);
// tmax, the maximum time between metrics
xdr_int(tmax);
// dmax, the maximum data value
xdr_int(dmax);
xdr_int(1);
/*Num of the entries in extra_value field for
Ganglia 3.1.x*/
xdr_string("GROUP");
/*Group attribute*/
xdr_string(groupName);
/*Group value*/
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
// Now we send out a message with the actual value.
// Technically, we only need to send out the metadata message once for
// each metric, but I don't want to have to record which metrics we did and
// did not send.
offset = 0;
// we are sending a string value
xdr_int(133);
// hostName
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// format field
xdr_string("%s");
// metric value
xdr_string(value);
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
public static Object waitForProxy(Class protocol, long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout, long timeout) throws IOException
long startTime = System.currentTimeMillis();
IOException ioe;
while (true) {
try {
return getProxy(protocol, clientVersion, addr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout);
} catch (ConnectException se) {
// namenode has not been started
LOG.info("Server at " + addr + " not available yet, Zzzzz...");
ioe = se;
} catch (SocketTimeoutException te) {
// namenode is busy
LOG.info("Problem connecting to server: " + addr);
ioe = te;
} catch (NoRouteToHostException nrthe) {
// perhaps a VIP is failing over
LOG.info("No route to host for server: " + addr);
ioe = nrthe;
}
// check if timed out
if (System.currentTimeMillis() - timeout >= startTime) {
throw ioe;
}
// wait for retry
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
// IGNORE
}
}
Even though tmpDir is dangling symlink to tmp, fullyDelete(tmpDir) should
delete tmpDir properly.
SATD_ADDED
setupDirs()
private void setupDirs() throws IOException
Assert.assertFalse(del.exists());
Assert.assertFalse(tmp.exists());
del.mkdirs();
tmp.mkdirs();
new File(del, FILE).createNewFile();
File tmpFile = new File(tmp, FILE);
tmpFile.createNewFile();
// create directories
dir1.mkdirs();
dir2.mkdirs();
new File(dir1, FILE).createNewFile();
new File(dir2, FILE).createNewFile();
// create a symlink to file
File link = new File(del, LINK);
FileUtil.symLink(tmpFile.toString(), link.toString());
// create a symlink to dir
File linkDir = new File(del, "tmpDir");
FileUtil.symLink(tmp.toString(), linkDir.toString());
Assert.assertEquals(5, del.listFiles().length);
Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y)
should delete 'y' properly.
SATD_ADDED
setupDirs()
private void setupDirs() throws IOException
Assert.assertFalse(del.exists());
Assert.assertFalse(tmp.exists());
del.mkdirs();
tmp.mkdirs();
new File(del, FILE).createNewFile();
File tmpFile = new File(tmp, FILE);
tmpFile.createNewFile();
// create directories
dir1.mkdirs();
dir2.mkdirs();
new File(dir1, FILE).createNewFile();
new File(dir2, FILE).createNewFile();
// create a symlink to file
File link = new File(del, LINK);
FileUtil.symLink(tmpFile.toString(), link.toString());
// create a symlink to dir
File linkDir = new File(del, "tmpDir");
FileUtil.symLink(tmp.toString(), linkDir.toString());
Assert.assertEquals(5, del.listFiles().length);
NOTE: this is the one place we do NOT want to save the number
of bytes sent (nRemaining here) into lastBytesSent: since we
are resending what we've already sent before, offset is nonzero
in general (only way it could be zero is if it already equals
nRemaining), which would then screw up the offset calculation
_next_ time around. IOW, getRemaining() is in terms of the
original, zero-offset bufferload, so lastBytesSent must be as
well. Cheesy ASCII art:
<------------ m, lastBytesSent ----------->
+===============================================+
buffer: |1111111111|22222222222222222|333333333333| |
+===============================================+
#1: <-- off -->|<-------- nRemaining --------->
#2: <----------- off ----------->|<-- nRem. -->
FIXME? Inflater docs say: 'it is also necessary to provide an extra
\"dummy\" byte as input. This is required by the ZLIB native
library in order to support certain optimizations.' However,
this does not appear to be true, and in any case, it's not
entirely clear where the byte should go or what its value
should be. Perhaps it suffices to have some deflated bytes
in the first buffer load? (But how else would one do it?)
SATD_ADDED
needsInput()
public synchronized boolean needsInput()
if (state == GzipStateLabel.DEFLATE_STREAM) {
// most common case
return inflater.needsInput();
}
// see userBufLen comment at top of decompress(); currently no need to
// verify userBufLen <= 0
return (state != GzipStateLabel.FINISHED);
// Do the authorization
if (HttpServer.hasAdministratorAccess(getServletContext(), request, response)) {
// Authorization is done. Just call super.
super.doGet(request, response);
}
Authorize the superuser which is doing doAs
@param user ugi of the effective or proxy user which contains a real user
@param remoteAddress the ip address of client
@param conf configuration
@throws AuthorizationException
public static synchronized void refresh(Configuration conf, PolicyProvider provider)
// Get the system property 'hadoop.policy.file'
String policyFile = System.getProperty("hadoop.policy.file", HADOOP_POLICY_FILE);
// Make a copy of the original config, and load the policy file
Configuration policyConf = new Configuration(conf);
policyConf.addResource(policyFile);
final Map, AccessControlList> newAcls = new IdentityHashMap, AccessControlList>();
// Parse the config file
Service[] services = provider.getServices();
if (services != null) {
for (Service service : services) {
AccessControlList acl = new AccessControlList(policyConf.get(service.getServiceKey(), AccessControlList.WILDCARD_ACL_VALUE));
newAcls.put(service.getProtocol(), acl);
}
}
// Flip to the newly parsed permissions
protocolToAcl = newAcls;
Asking the CodecPool for a decompressor for GzipCodec
should return null as well.
SATD_ADDED
testCodecPoolAndGzipDecompressor()
public void testCodecPoolAndGzipDecompressor()
// BuiltInZlibInflater should not be used as the GzipCodec decompressor.
// Assert that this is the case.
// Don't use native libs for this test.
Configuration conf = new Configuration();
conf.setBoolean("hadoop.native.lib", false);
assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf));
// This should give us a BuiltInZlibInflater.
Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
assertNotNull("zlibDecompressor is null!", zlibDecompressor);
assertTrue("ZlibFactory returned unexpected inflator", zlibDecompressor instanceof BuiltInZlibInflater);
// Asking for a decompressor directly from GzipCodec should return null;
// its createOutputStream() just wraps the existing stream in a
// java.util.zip.GZIPOutputStream.
CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
CompressionCodec codec = ccf.getCodec(new Path("foo.gz"));
assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec);
Decompressor codecDecompressor = codec.createDecompressor();
if (null != codecDecompressor) {
fail("Got non-null codecDecompressor: " + codecDecompressor);
}
// Asking the CodecPool for a decompressor for GzipCodec
// should return null as well.
Decompressor poolDecompressor = CodecPool.getDecompressor(codec);
if (null != poolDecompressor) {
fail("Got non-null poolDecompressor: " + poolDecompressor);
}
// If we then ensure that the pool is populated...
CodecPool.returnDecompressor(zlibDecompressor);
// Asking the pool another time should still not bind this to GzipCodec.
poolDecompressor = CodecPool.getDecompressor(codec);
if (null != poolDecompressor) {
fail("Second time, got non-null poolDecompressor: " + poolDecompressor);
}
Some filesystem like HDFS ignore the \"x\" bit if the permission.
Others like localFs does not.
Override the method below if the file system being tested masks our
certain bits for file masks.
Some filesystem like HDFS ignore the \"x\" bit if the permission.
Others like localFs does not.
Override the method below if the file system being tested masks our
certain bits for file masks.
This may have been a misparse. java.net.URI specifies that
a URI is of the form:
URI ::= [SCHEME-PART:]SCHEME-SPECIFIC-PART
The scheme-specific-part may be parsed in numerous ways, but if
it starts with a '/' character, that makes it a \"hierarchical URI\",
subject to the following parsing:
SCHEME-SPECIFIC-PART ::= \"//\" AUTHORITY-PART
AUTHORITY-PART ::= [USER-INFO-PART] HOSTNAME [ \":\" PORT ]
In Hadoop, we require a host-based authority as well.
java.net.URI parses left-to-right, so deprecated hostnames of the
form 'foo:8020' will have 'foo' as their scheme and '8020' as their
scheme-specific-part. We don't want this behavior.
unqualified is \"hdfs://\"
SATD_REMOVED
get(Configuration)
public static FileSystem get(Configuration conf) throws IOException
This may have been a misparse. java.net.URI specifies that
a URI is of the form:
URI ::= [SCHEME-PART:]SCHEME-SPECIFIC-PART
The scheme-specific-part may be parsed in numerous ways, but if
it starts with a '/' character, that makes it a \"hierarchical URI\",
subject to the following parsing:
SCHEME-SPECIFIC-PART ::= \"//\" AUTHORITY-PART
AUTHORITY-PART ::= [USER-INFO-PART] HOSTNAME [ \":\" PORT ]
In Hadoop, we require a host-based authority as well.
java.net.URI parses left-to-right, so deprecated hostnames of the
form 'foo:8020' will have 'foo' as their scheme and '8020' as their
scheme-specific-part. We don't want this behavior.
convert old-format name to new-format name
\"local\" is now \"file:///\".
SATD_REMOVED
get(Configuration)
public static FileSystem get(Configuration conf) throws IOException
This may have been a misparse. java.net.URI specifies that
a URI is of the form:
URI ::= [SCHEME-PART:]SCHEME-SPECIFIC-PART
The scheme-specific-part may be parsed in numerous ways, but if
it starts with a '/' character, that makes it a \"hierarchical URI\",
subject to the following parsing:
SCHEME-SPECIFIC-PART ::= \"//\" AUTHORITY-PART
AUTHORITY-PART ::= [USER-INFO-PART] HOSTNAME [ \":\" PORT ]
In Hadoop, we require a host-based authority as well.
java.net.URI parses left-to-right, so deprecated hostnames of the
form 'foo:8020' will have 'foo' as their scheme and '8020' as their
scheme-specific-part. We don't want this behavior.
SATD_ADDED
get(Configuration)
public static FileSystem get(Configuration conf) throws IOException
Some filesystem like HDFS ignore the \"x\" bit if the permission.
Others like localFs does not.
Override the method below if the file system being tested masks our
certain bits for file masks.
protected FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum, boolean createParent) throws IOException
checkPath(f);
// Default impl assumes that permissions do not matter
// calling the regular create is good enough.
// FSs that implement permissions should override this.
if (!createParent) {
// parent must exist.
// since this.create makes parent dirs automatically
// we must throw exception if parent does not exist.
final FileStatus stat = getFileStatus(f.getParent());
if (stat == null) {
throw new FileNotFoundException("Missing parent:" + f);
}
if (!stat.isDir()) {
throw new ParentNotDirectoryException("parent is not a dir:" + f);
}
// parent does exist - go ahead with create of file.
}
return fsImpl.primitiveCreate(f, absolutePermission, flag, bufferSize, replication, blockSize, progress, bytesPerChecksum);
Default impl is to assume that permissions do not matter and hence
calling the regular mkdirs is good enough.
FSs that implement permissions should override this.
SATD_ADDED
get(Configuration)
public static FileSystem get(Configuration conf) throws IOException
Default impl assumes that permissions do not matter and
nor does the bytesPerChecksum hence
calling the regular create is good enough.
FSs that implement permissions should override this.
SATD_ADDED
get(Configuration)
public static FileSystem get(Configuration conf) throws IOException
Default impl is to assume that permissions do not matter and hence
calling the regular mkdirs is good enough.
FSs that implement permissions should override this.
SATD_ADDED
get(Configuration)
public static FileSystem get(Configuration conf) throws IOException
Default impl assumes that permissions do not matter and
nor does the bytesPerChecksum hence
calling the regular create is good enough.
FSs that implement permissions should override this.
SATD_ADDED
get(Configuration)
public static FileSystem get(Configuration conf) throws IOException
The following if clause handles the following case:
Assume the following scenario in BZip2 compressed stream where
. represent compressed data.
.....[48 bit Block].....[48 bit Block].....[48 bit Block]...
........................[47 bits][1 bit].....[48 bit Block]...
................................^[Assume a Byte alignment here]
........................................^^[current position of stream]
.....................^^[We go back 10 Bytes in stream and find a Block marker]
........................................^^[We align at wrong position!]
...........................................................^^[While this pos is correct]
SATD_ADDED
createOutputStream(OutputStream)
public CompressionOutputStream createOutputStream(OutputStream out) throws IOException
The following XDR recipe was done through a careful reading of
gm_protocol.x in Ganglia 3.1 and carefully examining the output of
the gmetric utility with strace.
if (name == null) {
LOG.warn("Metric was emitted with no name.");
return;
} else if (value == null) {
LOG.warn("Metric name " + name + " was emitted with a null value.");
return;
} else if (type == null) {
LOG.warn("Metric name " + name + ", value " + value + " has no type.");
return;
}
LOG.debug("Emitting metric " + name + ", type " + type + ", value " + value + " from hostname" + hostName);
String units = getUnits(name);
if (units == null) {
LOG.warn("Metric name " + name + ", value " + value + " had 'null' units");
units = "";
}
int slope = getSlope(name);
int tmax = getTmax(name);
int dmax = getDmax(name);
offset = 0;
String groupName = name.substring(0, name.lastIndexOf("."));
// The following XDR recipe was done through a careful reading of
// gm_protocol.x in Ganglia 3.1 and carefully examining the output of
// the gmetric utility with strace.
// First we send out a metadata message
// metric_id = metadata_msg
xdr_int(128);
// hostname
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// metric type
xdr_string(type);
// metric name
xdr_string(name);
// units
xdr_string(units);
// slope
xdr_int(slope);
// tmax, the maximum time between metrics
xdr_int(tmax);
// dmax, the maximum data value
xdr_int(dmax);
xdr_int(1);
/*Num of the entries in extra_value field for
Ganglia 3.1.x*/
xdr_string("GROUP");
/*Group attribute*/
xdr_string(groupName);
/*Group value*/
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
// Now we send out a message with the actual value.
// Technically, we only need to send out the metadata message once for
// each metric, but I don't want to have to record which metrics we did and
// did not send.
offset = 0;
// we are sending a string value
xdr_int(133);
// hostName
xdr_string(hostName);
// metric name
xdr_string(name);
// spoof = False
xdr_int(0);
// format field
xdr_string("%s");
// metric value
xdr_string(value);
for (SocketAddress socketAddress : metricsServers) {
DatagramPacket packet = new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
}
TODO: sample the generated key/value records, and put the numbers below
SATD_ADDED
setUp()
public void setUp() throws IOException
skip = !(Algorithm.LZO.isSupported());
if (skip) {
System.out.println("Skipped");
}
// TODO: sample the generated key/value records, and put the numbers below
init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo", 2605, 2558);
if (!skip)
super.setUp();
public void testDeserialization() throws IOException
// Create a test BlockLocation
String[] names = { "one", "two" };
String[] hosts = { "three", "four" };
String[] topologyPaths = { "five", "six" };
long offset = 25l;
long length = 55l;
BlockLocation bl = new BlockLocation(names, hosts, topologyPaths, offset, length);
DataOutputBuffer dob = new DataOutputBuffer();
// Serialize it
try {
bl.write(dob);
} catch (IOException e) {
fail("Unable to serialize data: " + e.getMessage());
}
byte[] bytes = dob.getData();
DataInput da = new DataInputStream(new ByteArrayInputStream(bytes));
// Try to re-create the BlockLocation the same way as is done during
// deserialization
BlockLocation bl2 = new BlockLocation();
try {
bl2.readFields(da);
} catch (IOException e) {
fail("Unable to deserialize BlockLocation: " + e.getMessage());
}
// Check that we got back what we started with
verifyDeserialization(bl2.getHosts(), hosts);
verifyDeserialization(bl2.getNames(), names);
verifyDeserialization(bl2.getTopologyPaths(), topologyPaths);
assertEquals(bl2.getOffset(), offset);
assertEquals(bl2.getLength(), length);
Check that every permission has its sticky bit represented correctly
SATD_ADDED
testStickyBitToString()
public void testStickyBitToString()
// Check that every permission has its sticky bit represented correctly
for (boolean sb : new boolean[] { false, true }) {
for (FsAction u : FsAction.values()) {
for (FsAction g : FsAction.values()) {
for (FsAction o : FsAction.values()) {
FsPermission f = new FsPermission(u, g, o, sb);
if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('t', f.toString().charAt(8));
else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE))
assertEquals('T', f.toString().charAt(8));
else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('x', f.toString().charAt(8));
else
assertEquals('-', f.toString().charAt(8));
}
}
}
}
Refresh the service level authorization policy once again,
this time it should fail!
SATD_ADDED
testRefresh()
public void testRefresh() throws Exception
MiniDFSCluster dfs = null;
try {
final int slaves = 4;
// Turn on service-level authorization
Configuration conf = new Configuration();
conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class);
conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true);
// Start the mini dfs cluster
dfs = new MiniDFSCluster(conf, slaves, true, null);
// Refresh the service level authorization policy
refreshPolicy(conf);
// Simulate an 'edit' of hadoop-policy.xml
String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf");
File policyFile = new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE);
String policyFileCopy = ConfiguredPolicy.HADOOP_POLICY_FILE + ".orig";
// first save original
FileUtil.copy(// first save original
policyFile, // first save original
FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf);
rewriteHadoopPolicyFile(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
// Refresh the service level authorization policy
refreshPolicy(conf);
// Refresh the service level authorization policy once again,
// this time it should fail!
try {
// Note: hadoop-policy.xml for tests has
// security.refresh.policy.protocol.acl = ${user.name}
conf.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, UNKNOWN_USER);
refreshPolicy(conf);
fail("Refresh of NameNode's policy file cannot be successful!");
} catch (RemoteException re) {
System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re.unwrapRemoteException()));
} finally {
// Reset to original hadoop-policy.xml
FileUtil.fullyDelete(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
}
} finally {
if (dfs != null) {
dfs.shutdown();
}
}
MiniDFSCluster dfs = null;
try {
final int slaves = 4;
// Turn on service-level authorization
Configuration conf = new Configuration();
conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class);
conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, true);
// Start the mini dfs cluster
dfs = new MiniDFSCluster(conf, slaves, true, null);
// Refresh the service level authorization policy
refreshPolicy(conf);
// Simulate an 'edit' of hadoop-policy.xml
String confDir = System.getProperty("test.build.extraconf", "build/test/extraconf");
File policyFile = new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE);
String policyFileCopy = ConfiguredPolicy.HADOOP_POLICY_FILE + ".orig";
// first save original
FileUtil.copy(// first save original
policyFile, // first save original
FileSystem.getLocal(conf), new Path(confDir, policyFileCopy), false, conf);
rewriteHadoopPolicyFile(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
// Refresh the service level authorization policy
refreshPolicy(conf);
// Refresh the service level authorization policy once again,
// this time it should fail!
try {
// Note: hadoop-policy.xml for tests has
// security.refresh.policy.protocol.acl = ${user.name}
conf.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, UNKNOWN_USER);
refreshPolicy(conf);
fail("Refresh of NameNode's policy file cannot be successful!");
} catch (RemoteException re) {
System.out.println("Good, refresh worked... refresh failed with: " + StringUtils.stringifyException(re.unwrapRemoteException()));
} finally {
// Reset to original hadoop-policy.xml
FileUtil.fullyDelete(new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
}
} finally {
if (dfs != null) {
dfs.shutdown();
}
}
rare case where splits are exact, logs.length can be 4
SATD_ADDED
reset()
void reset()
final int oldsize = size;
do {
size = gen.nextInt(MAX_SIZE);
} while (oldsize == size);
final long oldseed = seed;
do {
seed = gen.nextLong() & Long.MAX_VALUE;
} while (oldseed == seed);
// Ask the user which local directory to upload
DirectoryDialog dialog = new DirectoryDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI);
dialog.setText("Select the local file or directory to upload");
String dirName = dialog.open();
final File dir = new File(dirName);
List files = new ArrayList();
files.add(dir);
// TODO enable upload command only when selection is exactly one folder
final List folders = filterSelection(DFSFolder.class, selection);
if (folders.size() >= 1)
uploadToDFS(folders.get(0), files);
// Ask the user which files to upload
FileDialog dialog = new FileDialog(Display.getCurrent().getActiveShell(), SWT.OPEN | SWT.MULTI);
dialog.setText("Select the local files to upload");
dialog.open();
List files = new ArrayList();
for (String fname : dialog.getFileNames()) files.add(new File(dialog.getFilterPath() + File.separator + fname));
// TODO enable upload command only when selection is exactly one folder
List folders = filterSelection(DFSFolder.class, selection);
if (folders.size() >= 1)
uploadToDFS(folders.get(0), files);
Check that every permission has its sticky bit represented correctly
SATD_ADDED
testStickyBitToString()
public void testStickyBitToString()
// Check that every permission has its sticky bit represented correctly
for (boolean sb : new boolean[] { false, true }) {
for (FsAction u : FsAction.values()) {
for (FsAction g : FsAction.values()) {
for (FsAction o : FsAction.values()) {
FsPermission f = new FsPermission(u, g, o, sb);
if (f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('t', f.toString().charAt(8));
else if (f.getStickyBit() && !f.getOtherAction().implies(EXECUTE))
assertEquals('T', f.toString().charAt(8));
else if (!f.getStickyBit() && f.getOtherAction().implies(EXECUTE))
assertEquals('x', f.toString().charAt(8));
else
assertEquals('-', f.toString().charAt(8));
}
}
}
}
TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb.
TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb.
Note that we *do* want to enable these tests with GERRIT_CHECK_NOTEDB, as
we need to be able to convert old, corrupt changes. However, those tests
don't necessarily need to pass.
SATD_REMOVED
setUp()
public void setUp() throws Exception
// TODO(dborowitz): Re-enable when ConsistencyChecker supports notedb.
// Note that we *do* want to enable these tests with GERRIT_CHECK_NOTEDB, as
// we need to be able to convert old, corrupt changes. However, those tests
// don't necessarily need to pass.
assume().that(notesMigration.enabled()).isFalse();
// Ignore client clone of project; repurpose as server-side TestRepository.
testRepo = new TestRepository<>((InMemoryRepository) repoManager.openRepository(project));
tip = testRepo.getRevWalk().parseCommit(testRepo.getRepository().exactRef("HEAD").getObjectId());
adminId = admin.getId();
checker = checkerProvider.get();
StringBuilder r = new StringBuilder();
r.append(RefNames.REFS_CHANGES);
int n = id.get();
int m = n % 100;
if (m < 10) {
r.append('0');
}
r.append(m);
r.append('/');
r.append(n);
r.append(RefNames.META_SUFFIX);
return r.toString();
We're not generally supposed to do work in provider constructors, but
this is a bit of a special case because we really need to have the ID
available by the time the dbInjector is created. This even applies during
RebuildNoteDb, which otherwise would have been a reasonable place to do
the ID generation. Fortunately, it's not much work, and it happens once.
StringBuilder s = new StringBuilder();
JsArray c = content();
for (int i = 0; i < c.length(); i++) {
Region r = c.get(i);
if (r.ab() != null) {
append(s, r.ab());
} else {
if (r.a() != null) {
append(s, r.a());
}
if (r.b() != null) {
append(s, r.b());
}
}
// TODO skip may need to be handled
}
return s.toString();
Every comment appears in both side maps as a linked pair.
It is only necessary to search one side to find a comment
on either side of the editor pair.
Every comment appears in both side maps as a linked pair.
It is only necessary to search one side to find a comment
on either side of the editor pair.
SATD_MOVED_FILE
commentNav(CodeMirror, Direction)
return new Runnable() {
@Override
public void run() {
// Every comment appears in both side maps as a linked pair.
// It is only necessary to search one side to find a comment
// on either side of the editor pair.
SortedMap map = map(src.side());
int line = src.extras().hasActiveLine() ? src.getLineNumber(src.extras().activeLine()) + 1 : 0;
if (dir == Direction.NEXT) {
map = map.tailMap(line + 1);
if (map.isEmpty()) {
return;
}
line = map.firstKey();
} else {
map = map.headMap(line);
if (map.isEmpty()) {
return;
}
line = map.lastKey();
}
SideBySideCommentGroup g = map.get(line);
if (g.getBoxCount() == 0) {
g = g.getPeer();
}
CodeMirror cm = g.getCm();
double y = cm.heightAtLine(g.getLine() - 1, "local");
cm.setCursor(Pos.create(g.getLine() - 1));
cm.scrollToY(y - 0.5 * cm.scrollbarV().getClientHeight());
cm.focus();
}
};
Runnable commentNav(final CodeMirror src, final Direction dir)
TODO(dborowitz): This should maybe be a synthetic timestamp just
after the actual last update in the history. On the one hand using
the commit updated time is reasonable, but on the other it might be
non-monotonic, and who knows what would break then.
Take advantage of an existing update on All-Users to prune any
published comments from drafts. NoteDbUpdateManager takes care of
ensuring that this update is applied before its dependent draft
update.
Deleting aggressively in this way, combined with filtering out
duplicate published/draft comments in ChangeNotes#getDraftComments,
makes up for the fact that updates between the change repo and
All-Users are not atomic.
TODO(dborowitz): We might want to distinguish between deleted
drafts that we're fixing up after the fact by putting them in a
separate commit. But note that we don't care much about the commit
graph of the draft ref, particularly because the ref is completely
deleted when all drafts are gone.
// Prohibit various kinds of illegal operations on comments.
Set existing = new HashSet<>();
for (RevisionNote rn : existingNotes.values()) {
for (PatchLineComment c : rn.comments) {
existing.add(c.getKey());
if (draftUpdate != null) {
// Take advantage of an existing update on All-Users to prune any
// published comments from drafts. NoteDbUpdateManager takes care of
// ensuring that this update is applied before its dependent draft
// update.
//
// Deleting aggressively in this way, combined with filtering out
// duplicate published/draft comments in ChangeNotes#getDraftComments,
// makes up for the fact that updates between the change repo and
// All-Users are not atomic.
//
// TODO(dborowitz): We might want to distinguish between deleted
// drafts that we're fixing up after the fact by putting them in a
// separate commit. But note that we don't care much about the commit
// graph of the draft ref, particularly because the ref is completely
// deleted when all drafts are gone.
draftUpdate.deleteComment(c.getRevId(), c.getKey());
}
}
}
for (RevisionNoteBuilder b : toUpdate.values()) {
for (PatchLineComment c : b.put.values()) {
if (existing.contains(c.getKey())) {
throw new OrmException("Cannot update existing published comment: " + c);
}
}
}
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
CLASS_OR_METHOD_CHANGED
rebuildAsync(Change.Id, ListeningExecutorService)
public ListenableFuture rebuildAsync(final Change.Id id, ListeningExecutorService executor)
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
// TODO - dborowitz: add NEW_CHANGE type for default.
ChangeKind kind = ChangeKind.REWORK;
// Trivial case: if we're on the first patch, we don't need to open
// the repository.
if (patch.getId().get() > 1) {
try (Repository repo = repoManager.openRepository(notes.getProjectName())) {
ProjectState projectState = projectCache.checkedGet(notes.getProjectName());
ChangeData cd = changeDataFactory.create(db, notes);
Collection patchSetCollection = cd.patchSets();
PatchSet priorPs = patch;
for (PatchSet ps : patchSetCollection) {
if (ps.getId().get() < patch.getId().get() && (ps.getId().get() > priorPs.getId().get() || priorPs == patch)) {
// We only want the previous patch set, so walk until the last one
priorPs = ps;
}
}
// If we still think the previous patch is the current patch,
// we only have one patch set. Return the default.
// This can happen if a user creates a draft, uploads a second patch,
// and deletes the draft.
if (priorPs != patch) {
kind = cache.getChangeKind(projectState, repo, ObjectId.fromString(priorPs.getRevision().get()), ObjectId.fromString(patch.getRevision().get()));
}
} catch (IOException | OrmException e) {
// Do nothing; assume we have a complex change
log.warn("Unable to get change kind for patchSet " + patch.getPatchSetId() + "of change " + notes.getChangeId(), e);
}
}
return kind;
final UserIdentity u = new UserIdentity();
u.setName(who.getName());
u.setEmail(who.getEmailAddress());
u.setDate(new Timestamp(who.getWhen().getTime()));
u.setTimeZone(who.getTimeZoneOffset());
// If only one account has access to this email address, select it
// as the identity of the user.
//
final Set a = byEmailCache.get(u.getEmail());
if (a.size() == 1) {
u.setAccount(a.iterator().next());
}
return u;
UserIdentity u = new UserIdentity();
u.setName(who.getName());
u.setEmail(who.getEmailAddress());
u.setDate(new Timestamp(who.getWhen().getTime()));
u.setTimeZone(who.getTimeZoneOffset());
// If only one account has access to this email address, select it
// as the identity of the user.
//
Set a = byEmailCache.get(u.getEmail());
if (a.size() == 1) {
u.setAccount(a.iterator().next());
}
return u;
// TODO - dborowitz: add NEW_CHANGE type for default.
ChangeKind kind = ChangeKind.REWORK;
// Trivial case: if we're on the first patch, we don't need to open
// the repository.
if (patch.getId().get() > 1) {
try (Repository repo = repoManager.openRepository(notes.getProjectName())) {
ProjectState projectState = projectCache.checkedGet(notes.getProjectName());
ChangeData cd = changeDataFactory.create(db, notes);
Collection patchSetCollection = cd.patchSets();
PatchSet priorPs = patch;
for (PatchSet ps : patchSetCollection) {
if (ps.getId().get() < patch.getId().get() && (ps.getId().get() > priorPs.getId().get() || priorPs == patch)) {
// We only want the previous patch set, so walk until the last one
priorPs = ps;
}
}
// If we still think the previous patch is the current patch,
// we only have one patch set. Return the default.
// This can happen if a user creates a draft, uploads a second patch,
// and deletes the draft.
if (priorPs != patch) {
kind = cache.getChangeKind(projectState, repo, ObjectId.fromString(priorPs.getRevision().get()), ObjectId.fromString(patch.getRevision().get()));
}
} catch (IOException | OrmException e) {
// Do nothing; assume we have a complex change
log.warn("Unable to get change kind for patchSet " + patch.getPatchSetId() + "of change " + notes.getChangeId(), e);
}
}
return kind;
ignore non valid numbers
We don't want to popup another ugly dialog just to say
\"The number you've provided is invalid, try again\"
SATD_ADDED
gotoLine()
private Runnable gotoLine()
return new Runnable() {
@Override
public void run() {
String n = Window.prompt(EditConstants.I.gotoLineNumber(), "");
if (n != null) {
try {
int line = Integer.parseInt(n);
line--;
if (line >= 0) {
cm.scrollToLine(line);
}
} catch (NumberFormatException e) {
// ignore non valid numbers
// We don't want to popup another ugly dialog just to say
// "The number you've provided is invalid, try again"
}
}
}
};
If too many changes failed, maybe there was a bug in the indexer. Don't
trust the results. This is not an exact percentage since we bump the same
failure counter if a project can't be read, but close enough.
TODO Don't load the changes directly from the database, but provide
project name + change ID to changeDataFactory, or delete this predicate.
SATD_ADDED
read()
public ResultSet read() throws OrmException
Set ids = new HashSet<>();
for (PatchLineComment sc : args.plcUtil.draftByAuthor(args.db.get(), accountId)) {
ids.add(sc.getKey().getParentKey().getParentKey().getParentKey());
}
List r = new ArrayList<>(ids.size());
// TODO Don't load the changes directly from the database, but provide
// project name + change ID to changeDataFactory, or delete this predicate.
for (Change c : args.db.get().changes().get(ids)) {
r.add(args.changeDataFactory.create(args.db.get(), c));
}
return new ListResultSet<>(r);
Just don't BCC everyone. Better to send a partial message to those
we already have queued up then to fail deliver entirely to people
who have a lower interest in the change.
Just don't BCC everyone. Better to send a partial message to those
we already have queued up then to fail deliver entirely to people
who have a lower interest in the change.
CLASS_OR_METHOD_CHANGED
setFrom(Account.Id)
public void setFrom(final Account.Id id)
super.setFrom(id);
/**
* Is the from user in an email squelching group?
*/
final IdentifiedUser user = args.identifiedUserFactory.create(id);
emailOnlyAuthors = !user.getCapabilities().canEmailReviewers();
private ChangeData toChangeData(Document doc, Set fields, String idFieldName)
ChangeData cd;
// Either change or the ID field was guaranteed to be included in the call
// to fields() above.
BytesRef cb = doc.getBinaryValue(CHANGE_FIELD);
if (cb != null) {
cd = changeDataFactory.create(db.get(), ChangeProtoField.CODEC.decode(cb.bytes, cb.offset, cb.length));
} else {
int id = doc.getField(idFieldName).numericValue().intValue();
// TODO(ekempin): Pass project to changeDataFactory
@SuppressWarnings("unused")
Project.NameKey project = new Project.NameKey(doc.getField(PROJECT.getName()).stringValue());
cd = changeDataFactory.create(db.get(), new Change.Id(id));
}
if (fields.contains(PATCH_SET_FIELD)) {
decodePatchSets(doc, cd);
}
if (fields.contains(APPROVAL_FIELD)) {
decodeApprovals(doc, cd);
}
if (fields.contains(ADDED_FIELD) && fields.contains(DELETED_FIELD)) {
decodeChangedLines(doc, cd);
}
if (fields.contains(MERGEABLE_FIELD)) {
decodeMergeable(doc, cd);
}
if (fields.contains(REVIEWEDBY_FIELD)) {
decodeReviewedBy(doc, cd);
}
return cd;
public static GroupCollector create(Multimap changeRefsById, final ReviewDb db, final PatchSetUtil psUtil, final ChangeNotes.Factory notesFactory, final Project.NameKey project)
return new GroupCollector(transformRefs(changeRefsById), new Lookup() {
@Override
public List lookup(PatchSet.Id psId) throws OrmException {
// TODO(dborowitz): Reuse open repository from caller.
ChangeNotes notes = notesFactory.create(db, project, psId.getParentKey());
PatchSet ps = psUtil.get(db, notes, psId);
return ps != null ? ps.getGroups() : null;
}
});
for (CodeReviewCommit c : commits) {
// TODO(dborowitz): Seems like this could get expensive for many patch
// sets. Is there a more efficient implementation?
if (rw.isMergedInto(c, tip)) {
return c;
}
}
return null;
No patch set for the already merged commit, although we know it came form
a patch set ref. Fix up the database. Note that this uses the current
user as the uploader, which is as good a guess as any.
PatchSet.Id psId = alreadyMerged.getPatchsetId();
logDebug("Fixing up already-merged patch set {}", psId);
PatchSet prevPs = args.psUtil.current(ctx.getDb(), ctx.getNotes());
ctx.getRevWalk().parseBody(alreadyMerged);
ctx.getChange().setCurrentPatchSet(psId, alreadyMerged.getShortMessage(), ctx.getChange().getOriginalSubject());
PatchSet existing = args.psUtil.get(ctx.getDb(), ctx.getNotes(), psId);
if (existing != null) {
logDebug("Patch set row exists, only updating change");
return existing;
}
// No patch set for the already merged commit, although we know it came form
// a patch set ref. Fix up the database. Note that this uses the current
// user as the uploader, which is as good a guess as any.
List groups = prevPs != null ? prevPs.getGroups() : GroupCollector.getDefaultGroups(alreadyMerged);
return args.psUtil.insert(ctx.getDb(), ctx.getRevWalk(), ctx.getUpdate(psId), psId, alreadyMerged, false, groups, null);
Change c = ctx.getChange();
ReviewDb db = ctx.getDb();
logDebug("Setting change {} merged", c.getId());
// TODO(dborowitz): Use PatchSetUtil? But we don't have a recent notes.
mergedPatchSet = db.patchSets().get(c.currentPatchSetId());
c.setStatus(Change.Status.MERGED);
c.setSubmissionId(args.submissionId);
ctx.saveChange();
// TODO(dborowitz): We need to be able to change the author of the message,
// which is not the user from the update context. addMergedMessage was able
// to do this in the past.
if (msg != null) {
args.cmUtil.addChangeMessage(db, ctx.getUpdate(msg.getPatchSetId()), msg);
}
TODO(dborowitz): Pre-BatchUpdate behavior wrote the merged message on
the old patch set ID, so that's what we do here. I don't think this
was intentional, and it should be changed.
String uuid;
try {
uuid = ChangeUtil.messageUUID(ctx.getDb());
} catch (OrmException e) {
return null;
}
ChangeMessage m = new ChangeMessage(new ChangeMessage.Key(ctx.getChange().getId(), uuid), // TODO(dborowitz): Pre-BatchUpdate behavior wrote the merged message on
// the old patch set ID, so that's what we do here. I don't think this
// was intentional, and it should be changed.
null, ctx.getWhen(), toMerge.change().currentPatchSetId());
m.setMessage(body);
return m;
TODO(dborowitz): Can't use repo from ctx due to canMergeFlag.
TODO(dborowitz): Can't use repo from ctx due to canMergeFlag.
CLASS_OR_METHOD_CHANGED
updateRepoImpl(RepoContext)
public void updateRepoImpl(RepoContext ctx) throws IntegrationException, IOException
// There are multiple parents, so this is a merge commit. We don't want
// to rebase the merge as clients can't easily rebase their history with
// that merge present and replaced by an equivalent merge with a different
// first parent. So instead behave as though MERGE_IF_NECESSARY was
// configured.
MergeTip mergeTip = args.mergeTip;
if (args.rw.isMergedInto(mergeTip.getCurrentTip(), toMerge)) {
mergeTip.moveTipTo(toMerge, toMerge);
acceptMergeTip(mergeTip);
} else {
// TODO(dborowitz): Can't use repo from ctx due to canMergeFlag.
CodeReviewCommit newTip = args.mergeUtil.mergeOneCommit(args.serverIdent, args.serverIdent, args.repo, args.rw, args.inserter, args.canMergeFlag, args.destBranch, mergeTip.getCurrentTip(), toMerge);
mergeTip.moveTipTo(newTip, toMerge);
}
args.mergeUtil.markCleanMerges(args.rw, args.canMergeFlag, mergeTip.getCurrentTip(), args.alreadyAccepted);
acceptMergeTip(mergeTip);
logDebug("Setting change {} merged", c.getId());
ChangeUpdate update = null;
final PatchSetApproval submitter;
PatchSet merged;
try {
db.changes().beginTransaction(c.getId());
// We must pull the patchset out of commits, because the patchset ID is
// modified when using the cherry-pick merge strategy.
CodeReviewCommit commit = commits.get(c.getId());
PatchSet.Id mergedId = commit.change().currentPatchSetId();
// TODO(dborowitz): Use PatchSetUtil after BatchUpdate migration.
merged = db.patchSets().get(mergedId);
c = setMergedPatchSet(c.getId(), mergedId);
submitter = approvalsUtil.getSubmitter(db, commit.notes(), mergedId);
ChangeControl control = commit.getControl();
update = updateFactory.create(control, c.getLastUpdatedOn());
// TODO(yyonas): we need to be able to change the author of the message
// is not the person for whom the change was made. addMergedMessage
// did this in the past.
if (msg != null) {
cmUtil.addChangeMessage(db, update, msg);
}
db.commit();
} finally {
db.rollback();
}
update.commit();
indexer.index(db, c);
try {
mergedSenderFactory.create(c.getId(), submitter != null ? submitter.getAccountId() : null).sendAsync();
} catch (Exception e) {
log.error("Cannot email merged notification for " + c.getId(), e);
}
if (submitter != null && mergeResultRev != null) {
try {
hooks.doChangeMergedHook(c, accountCache.get(submitter.getAccountId()).getAccount(), merged, db, mergeResultRev.name());
} catch (OrmException ex) {
logError("Cannot run hook for submitted patch set " + c.getId(), ex);
}
}
This seems like a cheap trick. It doesn't properly account for a
file that gets renamed between patch set 1 and patch set 2. We
will wind up packing the wrong Patch object because we didn't do
proper rename detection between the patch sets.
This seems like a cheap trick. It doesn't properly account for a
file that gets renamed between patch set 1 and patch set 2. We
will wind up packing the wrong Patch object because we didn't do
proper rename detection between the patch sets.
Map byKey = new HashMap<>();
if (loadHistory) {
// This seems like a cheap trick. It doesn't properly account for a
// file that gets renamed between patch set 1 and patch set 2. We
// will wind up packing the wrong Patch object because we didn't do
// proper rename detection between the patch sets.
//
history = new ArrayList<>();
for (PatchSet ps : psUtil.byChange(db, notes)) {
if (!control.isPatchVisible(ps, db)) {
continue;
}
String name = fileName;
if (psa != null) {
switch(changeType) {
case COPIED:
case RENAMED:
if (ps.getId().equals(psa)) {
name = oldName;
}
break;
case MODIFIED:
case DELETED:
case ADDED:
case REWRITE:
break;
}
}
Patch p = new Patch(new Patch.Key(ps.getId(), name));
history.add(p);
byKey.put(p.getKey(), p);
}
if (edit != null && edit.isPresent()) {
Patch p = new Patch(new Patch.Key(new PatchSet.Id(psb.getParentKey(), 0), fileName));
history.add(p);
byKey.put(p.getKey(), p);
}
}
if (loadComments && edit == null) {
AccountInfoCacheFactory aic = aicFactory.create();
comments = new CommentDetail(psa, psb);
switch(changeType) {
case ADDED:
case MODIFIED:
loadPublished(byKey, aic, newName);
break;
case DELETED:
loadPublished(byKey, aic, newName);
break;
case COPIED:
case RENAMED:
if (psa != null) {
loadPublished(byKey, aic, oldName);
}
loadPublished(byKey, aic, newName);
break;
case REWRITE:
break;
}
CurrentUser user = control.getUser();
if (user.isIdentifiedUser()) {
Account.Id me = user.getAccountId();
switch(changeType) {
case ADDED:
case MODIFIED:
loadDrafts(byKey, aic, me, newName);
break;
case DELETED:
loadDrafts(byKey, aic, me, newName);
break;
case COPIED:
case RENAMED:
if (psa != null) {
loadDrafts(byKey, aic, me, oldName);
}
loadDrafts(byKey, aic, me, newName);
break;
case REWRITE:
break;
}
}
comments.setAccountInfoCache(aic.create());
}
logDebug("Validating {} changes", submitted.size());
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new IntegrationException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new IntegrationException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new IntegrationException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId(), caller);
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
continue;
}
commit.add(canMergeFlag);
toSubmit.put(submitType, cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return toSubmit;
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
logDebug("Validating {} changes", submitted.size());
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new IntegrationException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new IntegrationException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new IntegrationException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId(), caller);
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
continue;
}
commit.add(canMergeFlag);
toSubmit.put(submitType, cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return toSubmit;
TODO(dborowitz): Can't use repo from ctx due to canMergeFlag.
SATD_ADDED
updateRepo(RepoContext)
public void updateRepo(RepoContext ctx) throws IntegrationException, IOException
// There are multiple parents, so this is a merge commit. We don't want
// to rebase the merge as clients can't easily rebase their history with
// that merge present and replaced by an equivalent merge with a different
// first parent. So instead behave as though MERGE_IF_NECESSARY was
// configured.
if (args.rw.isMergedInto(mergeTip.getCurrentTip(), toMerge)) {
mergeTip.moveTipTo(toMerge, toMerge);
acceptMergeTip(mergeTip);
} else {
PersonIdent myIdent = args.serverIdent.get();
// TODO(dborowitz): Can't use repo from ctx due to canMergeFlag.
CodeReviewCommit newTip = args.mergeUtil.mergeOneCommit(myIdent, myIdent, args.repo, args.rw, args.inserter, args.canMergeFlag, args.destBranch, mergeTip.getCurrentTip(), toMerge);
mergeTip.moveTipTo(newTip, toMerge);
}
args.mergeUtil.markCleanMerges(args.rw, args.canMergeFlag, mergeTip.getCurrentTip(), args.alreadyAccepted);
acceptMergeTip(mergeTip);
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
logDebug("Validating {} changes", submitted.size());
List toSubmit = new ArrayList<>(submitted.size());
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new IntegrationException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
SubmitType submitType = null;
ChangeData choseSubmitTypeFrom = null;
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new IntegrationException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new IntegrationException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType st = getSubmitType(commit.getControl(), ps);
if (st == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
throw new IntegrationException("No submit type for change " + cd.getId());
}
if (submitType == null) {
submitType = st;
choseSubmitTypeFrom = cd;
} else if (st != submitType) {
throw new IntegrationException(String.format("Change %s has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", cd.getId(), st, submitType, choseSubmitTypeFrom.getId()));
}
commit.add(canMergeFlag);
toSubmit.add(cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
logDebug("Validating {} changes", submitted.size());
List toSubmit = new ArrayList<>(submitted.size());
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new IntegrationException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
SubmitType submitType = null;
ChangeData choseSubmitTypeFrom = null;
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new IntegrationException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new IntegrationException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType st = getSubmitType(commit.getControl(), ps);
if (st == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
throw new IntegrationException("No submit type for change " + cd.getId());
}
if (submitType == null) {
submitType = st;
choseSubmitTypeFrom = cd;
} else if (st != submitType) {
throw new IntegrationException(String.format("Change %s has submit type %s, but previously chose submit type %s " + "from change %s in the same batch", cd.getId(), st, submitType, choseSubmitTypeFrom.getId()));
}
commit.add(canMergeFlag);
toSubmit.add(cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return new AutoValue_MergeOp_BranchBatch(submitType, toSubmit);
com.google.gerrit.server.account.AuthRequest areq = new com.google.gerrit.server.account.AuthRequest(user.getExternalId());
AuthResult arsp = null;
try {
String claimedIdentifier = user.getClaimedIdentity();
Account.Id actualId = accountManager.lookup(user.getExternalId());
Account.Id claimedId = null;
// We try to retrieve claimed identity.
// For some reason, for example staging instance
// it may deviate from the really old OpenID identity.
// What we want to avoid in any event is to create new
// account instead of linking to the existing one.
// That why we query it here, not to lose linking mode.
if (!Strings.isNullOrEmpty(claimedIdentifier)) {
claimedId = accountManager.lookup(claimedIdentifier);
if (claimedId == null) {
log.debug("Claimed identity is unknown");
}
}
// Use case 1: claimed identity was provided during handshake phase
// and user account exists for this identity
if (claimedId != null) {
log.debug("Claimed identity is set and is known");
if (actualId != null) {
if (claimedId.equals(actualId)) {
// Both link to the same account, that's what we expected.
} else {
// This is (for now) a fatal error. There are two records
// for what might be the same user. The admin would have to
// link the accounts manually.
log.error("OAuth accounts disagree over user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier + "\n" + " Delgate ID: " + actualId + " is " + user.getExternalId());
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
} else {
// Claimed account already exists: link to it.
//
try {
accountManager.link(claimedId, areq);
} catch (OrmException e) {
log.error("Cannot link: " + user.getExternalId() + " to user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier);
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
}
} else if (linkMode) {
// Use case 2: link mode activated from the UI
try {
accountManager.link(identifiedUser.get().getAccountId(), areq);
} catch (OrmException e) {
log.error("Cannot link: " + user.getExternalId() + " to user identity: " + identifiedUser.get().getAccountId());
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
} finally {
linkMode = false;
}
}
areq.setUserName(user.getUserName());
areq.setEmailAddress(user.getEmailAddress());
areq.setDisplayName(user.getDisplayName());
arsp = accountManager.authenticate(areq);
} catch (AccountException e) {
log.error("Unable to authenticate user \"" + user + "\"", e);
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
webSession.get().login(arsp, true);
StringBuilder rdr = new StringBuilder(urlProvider.get(req));
rdr.append(Url.decode(redirectToken));
rsp.sendRedirect(rdr.toString());
We try to retrieve claimed identity.
For some reason, for example staging instance
it may deviate from the really old OpenID identity.
What we want to avoid in any event is to create new
account instead of linking to the existing one.
That why we query it here, not to lose linking mode.
com.google.gerrit.server.account.AuthRequest areq = new com.google.gerrit.server.account.AuthRequest(user.getExternalId());
AuthResult arsp = null;
try {
String claimedIdentifier = user.getClaimedIdentity();
Account.Id actualId = accountManager.lookup(user.getExternalId());
Account.Id claimedId = null;
// We try to retrieve claimed identity.
// For some reason, for example staging instance
// it may deviate from the really old OpenID identity.
// What we want to avoid in any event is to create new
// account instead of linking to the existing one.
// That why we query it here, not to lose linking mode.
if (!Strings.isNullOrEmpty(claimedIdentifier)) {
claimedId = accountManager.lookup(claimedIdentifier);
if (claimedId == null) {
log.debug("Claimed identity is unknown");
}
}
// Use case 1: claimed identity was provided during handshake phase
// and user account exists for this identity
if (claimedId != null) {
log.debug("Claimed identity is set and is known");
if (actualId != null) {
if (claimedId.equals(actualId)) {
// Both link to the same account, that's what we expected.
} else {
// This is (for now) a fatal error. There are two records
// for what might be the same user. The admin would have to
// link the accounts manually.
log.error("OAuth accounts disagree over user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier + "\n" + " Delgate ID: " + actualId + " is " + user.getExternalId());
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
} else {
// Claimed account already exists: link to it.
//
try {
accountManager.link(claimedId, areq);
} catch (OrmException e) {
log.error("Cannot link: " + user.getExternalId() + " to user identity:\n" + " Claimed ID: " + claimedId + " is " + claimedIdentifier);
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
}
} else if (linkMode) {
// Use case 2: link mode activated from the UI
try {
accountManager.link(identifiedUser.get().getAccountId(), areq);
} catch (OrmException e) {
log.error("Cannot link: " + user.getExternalId() + " to user identity: " + identifiedUser.get().getAccountId());
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
} finally {
linkMode = false;
}
}
areq.setUserName(user.getUserName());
areq.setEmailAddress(user.getEmailAddress());
areq.setDisplayName(user.getDisplayName());
arsp = accountManager.authenticate(areq);
} catch (AccountException e) {
log.error("Unable to authenticate user \"" + user + "\"", e);
rsp.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
webSession.get().login(arsp, true);
StringBuilder rdr = new StringBuilder(urlProvider.get(req));
rdr.append(Url.decode(redirectToken));
rsp.sendRedirect(rdr.toString());
no providerId in the cookie value => assume default provider
note: a leading/trailing at sign is considered to belong to
the access token rather than being a separator
SATD_ADDED
init(FilterConfig)
public void init(FilterConfig config) throws ServletException
if (Strings.isNullOrEmpty(gitOAuthProvider)) {
pickOnlyProvider();
} else {
pickConfiguredProvider();
}
limit the number of query terms otherwise Gerrit will barf
SATD_ADDED
label(String)
public Predicate label(String name) throws QueryParseException, OrmException
Set accounts = null;
AccountGroup.UUID group = null;
// Parse for:
// label:CodeReview=1,user=jsmith or
// label:CodeReview=1,jsmith or
// label:CodeReview=1,group=android_approvers or
// label:CodeReview=1,android_approvers
// user/groups without a label will first attempt to match user
String[] splitReviewer = name.split(",", 2);
// remove all but the vote piece, e.g.'CodeReview=1'
name = splitReviewer[0];
if (splitReviewer.length == 2) {
// process the user/group piece
PredicateArgs lblArgs = new PredicateArgs(splitReviewer[1]);
for (Map.Entry pair : lblArgs.keyValue.entrySet()) {
if (pair.getKey().equalsIgnoreCase(ARG_ID_USER)) {
accounts = parseAccount(pair.getValue());
} else if (pair.getKey().equalsIgnoreCase(ARG_ID_GROUP)) {
group = parseGroup(pair.getValue()).getUUID();
} else {
throw new QueryParseException("Invalid argument identifier '" + pair.getKey() + "'");
}
}
for (String value : lblArgs.positional) {
if (accounts != null || group != null) {
throw new QueryParseException("more than one user/group specified (" + value + ")");
}
try {
accounts = parseAccount(value);
} catch (QueryParseException qpex) {
// If it doesn't match an account, see if it matches a group
// (accounts get precedence)
try {
group = parseGroup(value).getUUID();
} catch (QueryParseException e) {
throw error("Neither user nor group " + value + " found");
}
}
}
}
// expand a group predicate into multiple user predicates
if (group != null) {
Set allMembers = getMemberIds(group, new HashSet());
int maxTerms = args.indexConfig.maxLimit();
if (allMembers.size() > maxTerms) {
// limit the number of query terms otherwise Gerrit will barf
accounts = ImmutableSet.copyOf(Iterables.limit(allMembers, maxTerms));
} else {
accounts = allMembers;
}
}
return new LabelPredicate(args.projectCache, args.changeControlGenericFactory, args.userFactory, args.db, name, accounts, group);
Serve static resources directly from our JAR. This way we don't
need to unpack them into yet another temporary directory prior to
serving to clients.
This is the path we are accessed by clients within our domain.
SATD_REMOVED
makeContext(String, JettyEnv, Config)
private ContextHandler makeContext(final String contextPath, final JettyEnv env, final Config cfg)
final ServletContextHandler app = new ServletContextHandler();
// This enables the use of sessions in Jetty, feature available
// for Gerrit plug-ins to enable user-level sessions.
//
app.setSessionHandler(new SessionHandler());
app.setErrorHandler(new HiddenErrorHandler());
// This is the path we are accessed by clients within our domain.
//
app.setContextPath(contextPath);
// HTTP front-end filter to be used as surrogate of Apache HTTP
// reverse-proxy filtering.
// It is meant to be used as simpler tiny deployment of custom-made
// security enforcement (Security tokens, IP-based security filtering, others)
String filterClassName = cfg.getString("httpd", null, "filterClass");
if (filterClassName != null) {
try {
@SuppressWarnings("unchecked")
Class filterClass = (Class) Class.forName(filterClassName);
Filter filter = env.webInjector.getInstance(filterClass);
app.addFilter(new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC));
} catch (Throwable e) {
String errorMessage = "Unable to instantiate front-end HTTP Filter " + filterClassName;
log.error(errorMessage, e);
throw new IllegalArgumentException(errorMessage, e);
}
}
// Perform the same binding as our web.xml would do, but instead
// of using the listener to create the injector pass the one we
// already have built.
//
GuiceFilter filter = env.webInjector.getInstance(GuiceFilter.class);
app.addFilter(new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC));
app.addEventListener(new GuiceServletContextListener() {
@Override
protected Injector getInjector() {
return env.webInjector;
}
});
// Jetty requires at least one servlet be bound before it will
// bother running the filter above. Since the filter has all
// of our URLs except the static resources, the only servlet
// we need to bind is the default static resource servlet from
// the Jetty container.
//
final ServletHolder ds = app.addServlet(DefaultServlet.class, "/");
ds.setInitParameter("dirAllowed", "false");
ds.setInitParameter("redirectWelcome", "false");
ds.setInitParameter("useFileMappedBuffer", "false");
ds.setInitParameter("gzip", "true");
app.setWelcomeFiles(new String[0]);
return app;
private static List walkDescendentsImpl(ProjectControl ctl, Set alreadyEmittedChanges, ListMultimap children, List start) throws OrmException
if (start.isEmpty()) {
return ImmutableList.of();
}
Map maxPatchSetIds = new HashMap<>();
List allPatchSets = new ArrayList<>();
Deque pending = new ArrayDeque<>();
pending.addAll(start);
while (!pending.isEmpty()) {
PatchSetData psd = pending.remove();
if (!isVisible(psd, ctl)) {
continue;
}
if (!alreadyEmittedChanges.contains(psd.id())) {
// Don't emit anything for changes that were previously emitted, even
// though different patch sets might show up later. However, do
// continue walking through them for the purposes of finding indirect
// descendants.
PatchSet.Id oldMax = maxPatchSetIds.get(psd.id());
if (oldMax == null || psd.psId().get() > oldMax.get()) {
maxPatchSetIds.put(psd.id(), psd.psId());
}
allPatchSets.add(psd);
}
// Depth-first search with newest children first.
for (PatchSetData child : children.get(psd)) {
pending.addFirst(child);
}
}
// If we saw the same change multiple times, prefer the latest patch set.
List result = new ArrayList<>(allPatchSets.size());
for (PatchSetData psd : allPatchSets) {
if (checkNotNull(maxPatchSetIds.get(psd.id())).equals(psd.psId())) {
result.add(psd);
}
}
return result;
private static List walkDescendentsImpl(ProjectControl ctl, Set alreadyEmittedChanges, ListMultimap children, List start) throws OrmException
if (start.isEmpty()) {
return ImmutableList.of();
}
Map maxPatchSetIds = new HashMap<>();
List allPatchSets = new ArrayList<>();
Deque pending = new ArrayDeque<>();
pending.addAll(start);
while (!pending.isEmpty()) {
PatchSetData psd = pending.remove();
if (!isVisible(psd, ctl)) {
continue;
}
if (!alreadyEmittedChanges.contains(psd.id())) {
// Don't emit anything for changes that were previously emitted, even
// though different patch sets might show up later. However, do
// continue walking through them for the purposes of finding indirect
// descendants.
PatchSet.Id oldMax = maxPatchSetIds.get(psd.id());
if (oldMax == null || psd.psId().get() > oldMax.get()) {
maxPatchSetIds.put(psd.id(), psd.psId());
}
allPatchSets.add(psd);
}
// Breadth-first search with oldest children first.
// TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep
// parallel history together.
pending.addAll(Lists.reverse(children.get(psd)));
}
// If we saw the same change multiple times, prefer the latest patch set.
List result = new ArrayList<>(allPatchSets.size());
for (PatchSetData psd : allPatchSets) {
if (checkNotNull(maxPatchSetIds.get(psd.id())).equals(psd.psId())) {
result.add(psd);
}
}
return result;
private static List walkDescendentsImpl(ProjectControl ctl, Set alreadyEmittedChanges, ListMultimap children, List start) throws OrmException
if (start.isEmpty()) {
return ImmutableList.of();
}
Map maxPatchSetIds = new HashMap<>();
List allPatchSets = new ArrayList<>();
Deque pending = new ArrayDeque<>();
pending.addAll(start);
while (!pending.isEmpty()) {
PatchSetData psd = pending.remove();
if (!isVisible(psd, ctl)) {
continue;
}
if (!alreadyEmittedChanges.contains(psd.id())) {
// Don't emit anything for changes that were previously emitted, even
// though different patch sets might show up later. However, do
// continue walking through them for the purposes of finding indirect
// descendants.
PatchSet.Id oldMax = maxPatchSetIds.get(psd.id());
if (oldMax == null || psd.psId().get() > oldMax.get()) {
maxPatchSetIds.put(psd.id(), psd.psId());
}
allPatchSets.add(psd);
}
// Breadth-first search with oldest children first.
// TODO(dborowitz): After killing PatchSetAncestors, consider DFS to keep
// parallel history together.
pending.addAll(Lists.reverse(children.get(psd)));
}
// If we saw the same change multiple times, prefer the latest patch set.
List result = new ArrayList<>(allPatchSets.size());
for (PatchSetData psd : allPatchSets) {
if (checkNotNull(maxPatchSetIds.get(psd.id())).equals(psd.psId())) {
result.add(psd);
}
}
return result;
RFC4880 states:
\"If a key has been revoked because of a compromise, all signatures
created by that key are suspect. However, if it was merely superseded or
retired, old signatures are still valid.\"
Note that GnuPG does not implement this correctly, as it does not
consider the revocation reason and timestamp when checking whether a
signature (data or certification) is valid.
private static boolean isRevocationValid(PGPSignature revocation, RevocationReason reason, Date now)
// RFC4880 states:
// "If a key has been revoked because of a compromise, all signatures
// created by that key are suspect. However, if it was merely superseded or
// retired, old signatures are still valid."
//
// Note that GnuPG does not implement this correctly, as it does not
// consider the revocation reason and timestamp when checking whether a
// signature (data or certification) is valid.
return reason.getRevocationReason() == KEY_COMPROMISED || revocation.getCreationTime().before(now);
private void forceCallerAsReviewer(ChangeContext ctx, Map current, List ups, List del)
if (current.isEmpty() && ups.isEmpty()) {
// TODO Find another way to link reviewers to changes.
if (del.isEmpty()) {
// If no existing label is being set to 0, hack in the caller
// as a reviewer by picking the first server-wide LabelType.
PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(psId, user.getAccountId(), ctx.getChangeControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs());
c.setGranted(ctx.getWhen());
ups.add(c);
} else {
// Pick a random label that is about to be deleted and keep it.
Iterator i = del.iterator();
PatchSetApproval c = i.next();
c.setValue((short) 0);
c.setGranted(ctx.getWhen());
i.remove();
ups.add(c);
}
}
private void forceCallerAsReviewer(ChangeContext ctx, Map current, List ups, List del)
if (current.isEmpty() && ups.isEmpty()) {
// TODO Find another way to link reviewers to changes.
if (del.isEmpty()) {
// If no existing label is being set to 0, hack in the caller
// as a reviewer by picking the first server-wide LabelType.
PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(psId, user.getAccountId(), ctx.getChangeControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs());
c.setGranted(ctx.getWhen());
ups.add(c);
} else {
// Pick a random label that is about to be deleted and keep it.
Iterator i = del.iterator();
PatchSetApproval c = i.next();
c.setValue((short) 0);
c.setGranted(ctx.getWhen());
i.remove();
ups.add(c);
}
}
TODO(dborowitz): This is only public because callers expect validation to
happen before updating any refs, and they are still updating refs manually.
Make private once we have migrated ref updates into this class.
SATD_ADDED
validate()
public void validate() throws IOException, InvalidChangeOperationException
if (validated || validatePolicy == CommitValidators.Policy.NONE) {
return;
}
CommitValidators cv = commitValidatorsFactory.create(refControl, new NoSshInfo(), git);
String refName = patchSet.getId().toRefName();
CommitReceivedEvent event = new CommitReceivedEvent(new ReceiveCommand(ObjectId.zeroId(), commit.getId(), refName), refControl.getProjectControl().getProject(), refControl.getRefName(), commit, user);
try {
switch(validatePolicy) {
case RECEIVE_COMMITS:
NoteMap rejectCommits = BanCommit.loadRejectCommitsMap(git, revWalk);
cv.validateForReceiveCommits(event, rejectCommits);
break;
case GERRIT:
cv.validateForGerritCommits(event);
break;
case NONE:
break;
}
} catch (CommitValidationException e) {
throw new InvalidChangeOperationException(e.getMessage());
}
validated = true;
Only pushing magic branches: allow a valid push certificate even if the
key is not ultimately trusted. Assume anyone with Submit permission to
the branch is able to verify during review that the code is legitimate.
if (onlyMagicBranches(commands)) {
// Only pushing magic branches: allow a valid push certificate even if the
// key is not ultimately trusted. Assume anyone with Submit permission to
// the branch is able to verify during review that the code is legitimate.
return result.isOk();
} else {
// Directly updating one or more refs: require a trusted key.
return result.isTrusted();
}
This is not exactly the key stored in the store, but is equivalent. In
particular, it will have a Bouncy Castle version string. The armored
stream reader in PublicKeyStore doesn't give us an easy way to extract
the original ASCII armor.
This is not exactly the key stored in the store, but is equivalent. In
particular, it will have a Bouncy Castle version string. The armored
stream reader in PublicKeyStore doesn't give us an easy way to extract
the original ASCII armor.
CLASS_OR_METHOD_CHANGED
toJson(PGPPublicKey, CheckResult)
public static GpgKeyInfo toJson(PGPPublicKey key, CheckResult checkResult) throws IOException
GpgKeyInfo info = new GpgKeyInfo();
if (key != null) {
info.id = PublicKeyStore.keyIdToString(key.getKeyID());
info.fingerprint = Fingerprint.toString(key.getFingerprint());
@SuppressWarnings("unchecked")
Iterator userIds = key.getUserIDs();
info.userIds = ImmutableList.copyOf(userIds);
try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
ArmoredOutputStream aout = new ArmoredOutputStream(out)) {
// This is not exactly the key stored in the store, but is equivalent. In
// particular, it will have a Bouncy Castle version string. The armored
// stream reader in PublicKeyStore doesn't give us an easy way to extract
// the original ASCII armor.
key.encode(aout);
info.key = new String(out.toByteArray(), UTF_8);
}
}
info.status = checkResult.getStatus();
info.problems = checkResult.getProblems();
return info;
This is not exactly the key stored in the store, but is equivalent. In
particular, it will have a Bouncy Castle version string. The armored
stream reader in PublicKeyStore doesn't give us an easy way to extract
the original ASCII armor.
This is not exactly the key stored in the store, but is equivalent. In
particular, it will have a Bouncy Castle version string. The armored
stream reader in PublicKeyStore doesn't give us an easy way to extract
the original ASCII armor.
PGPPublicKey key = keyRing.getPublicKey();
GpgKeyInfo info = new GpgKeyInfo();
info.id = PublicKeyStore.keyIdToString(key.getKeyID());
info.fingerprint = Fingerprint.toString(key.getFingerprint());
@SuppressWarnings("unchecked")
Iterator userIds = key.getUserIDs();
info.userIds = ImmutableList.copyOf(userIds);
try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
ArmoredOutputStream aout = new ArmoredOutputStream(out)) {
// This is not exactly the key stored in the store, but is equivalent. In
// particular, it will have a Bouncy Castle version string. The armored
// stream reader in PublicKeyStore doesn't give us an easy way to extract
// the original ASCII armor.
key.encode(aout);
info.key = new String(out.toByteArray(), UTF_8);
}
CheckResult checkResult = checker.check(key, store);
info.status = checkResult.getStatus();
info.problems = checkResult.getProblems();
return info;
TODO(dborowitz): Not sure this is guaranteed in general.
TODO(dborowitz): Not sure this is guaranteed in general.
CLASS_OR_METHOD_CHANGED
addOp(ChangeControl, Op)
public BatchUpdate addOp(ChangeControl ctl, Op op)
Change.Id id = ctl.getChange().getId();
ChangeControl old = changeControls.get(id);
// TODO(dborowitz): Not sure this is guaranteed in general.
checkArgument(old == null || old == ctl, "mismatched ChangeControls for change %s", id);
ops.put(id, op);
changeControls.put(id, ctl);
return this;
TODO(dborowitz): Not sure this is guaranteed in general.
SATD_ADDED
addChangeOp(ChangeOp)
public BatchUpdate addChangeOp(ChangeOp op)
Change.Id id = op.ctl.getChange().getId();
ChangeControl old = changeControls.get(id);
// TODO(dborowitz): Not sure this is guaranteed in general.
checkArgument(old == null || old == op.ctl, "mismatched ChangeControls for change %s", id);
changeOps.put(id, op);
changeControls.put(id, op.ctl);
return this;
logDebug("Validating {} changes", submitted.size());
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new MergeException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new MergeException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new MergeException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
continue;
}
commit.add(canMergeFlag);
toSubmit.put(submitType, cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return toSubmit;
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
logDebug("Validating {} changes", submitted.size());
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new MergeException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new MergeException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new MergeException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
continue;
}
commit.add(canMergeFlag);
toSubmit.put(submitType, cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return toSubmit;
TODO(dborowitz): Merge this with AcceptanceTestRequestScope.
TODO(dborowitz): Merge this with AcceptanceTestRequestScope.
FILE_PATH_CHANGED
module()
static Module module()
return new AbstractModule() {
@Override
public void configure() {
install(new GerritRequestModule());
bind(RequestScopePropagator.class).to(Propagator.class);
bindScope(RequestScoped.class, InProcessProtocol.REQUEST);
}
@Provides
@RemotePeer
SocketAddress getSocketAddress() {
// TODO(dborowitz): Could potentially fake this with thread ID or
// something.
throw new OutOfScopeException("No remote peer in acceptance tests");
}
};
TODO(dborowitz): Swap out GitRepositoryManager somehow? Will probably be
necessary for notedb anyway.
SATD_ADDED
prepopulatedFields()
public void prepopulatedFields() throws Exception
assume().that(notesMigration.enabled()).isFalse();
TestRepository repo = createProject("repo");
Change change = newChange(repo, null, null, null, null).insert();
db = new DisabledReviewDb();
requestContext.setContext(newRequestContext(userId));
// Use QueryProcessor directly instead of API so we get ChangeDatas back.
List cds = queryProcessor.queryChanges(queryBuilder.parse(change.getId().toString())).changes();
assertThat(cds).hasSize(1);
ChangeData cd = cds.get(0);
cd.change();
cd.patchSets();
cd.currentApprovals();
cd.changedLines();
cd.reviewedBy();
// TODO(dborowitz): Swap out GitRepositoryManager somehow? Will probably be
// necessary for notedb anyway.
cd.isMergeable();
// Don't use ExpectedException since that wouldn't distinguish between
// failures here and on the previous calls.
try {
cd.messages();
} catch (AssertionError e) {
assertThat(e.getMessage()).isEqualTo(DisabledReviewDb.MESSAGE);
}
This is not exactly the key stored in the store, but is equivalent. In
particular, it will have a Bouncy Castle version string. The armored
stream reader in PublicKeyStore doesn't give us an easy way to extract
the original ASCII armor.
This is not exactly the key stored in the store, but is equivalent. In
particular, it will have a Bouncy Castle version string. The armored
stream reader in PublicKeyStore doesn't give us an easy way to extract
the original ASCII armor.
PGPPublicKey key = keyRing.getPublicKey();
GpgKeyInfo info = new GpgKeyInfo();
info.id = PublicKeyStore.keyIdToString(key.getKeyID());
info.fingerprint = Fingerprint.toString(key.getFingerprint());
@SuppressWarnings("unchecked")
Iterator userIds = key.getUserIDs();
info.userIds = ImmutableList.copyOf(userIds);
try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
ArmoredOutputStream aout = new ArmoredOutputStream(out)) {
// This is not exactly the key stored in the store, but is equivalent. In
// particular, it will have a Bouncy Castle version string. The armored
// stream reader in PublicKeyStore doesn't give us an easy way to extract
// the original ASCII armor.
key.encode(aout);
info.key = new String(out.toByteArray(), UTF_8);
}
return info;
TODO(dborowitz): Backoff and retry on LOCK_FAILURE.
TODO(dborowitz): Backoff and retry on LOCK_FAILURE.
FILE_PATH_CHANGED
storeKeys(AccountResource, List, Set)
private void storeKeys(AccountResource rsrc, List keyRings, Set toRemove) throws BadRequestException, ResourceConflictException, PGPException, IOException
try (PublicKeyStore store = storeProvider.get()) {
for (PGPPublicKeyRing keyRing : keyRings) {
PGPPublicKey key = keyRing.getPublicKey();
CheckResult result = checker.check(key);
if (!result.isOk()) {
throw new BadRequestException(String.format("Problems with public key %s:\n%s", keyToString(key), Joiner.on('\n').join(result.getProblems())));
}
store.add(keyRing);
}
for (Fingerprint fp : toRemove) {
store.remove(fp.get());
}
CommitBuilder cb = new CommitBuilder();
PersonIdent committer = serverIdent.get();
cb.setAuthor(rsrc.getUser().newCommitterIdent(committer.getWhen(), committer.getTimeZone()));
cb.setCommitter(committer);
RefUpdate.Result saveResult = store.save(cb);
switch(saveResult) {
case NEW:
case FAST_FORWARD:
case FORCED:
case NO_CHANGE:
break;
default:
// TODO(dborowitz): Backoff and retry on LOCK_FAILURE.
throw new ResourceConflictException("Failed to save public keys: " + saveResult);
}
}
TODO(dborowitz): Backoff and retry on LOCK_FAILURE.
TODO(dborowitz): Backoff and retry on LOCK_FAILURE.
CLASS_OR_METHOD_CHANGED
storeKeys(AccountResource, List, Set)
private void storeKeys(AccountResource rsrc, List keyRings, Set toRemove) throws BadRequestException, ResourceConflictException, PGPException, IOException
try (PublicKeyStore store = storeProvider.get()) {
for (PGPPublicKeyRing keyRing : keyRings) {
PGPPublicKey key = keyRing.getPublicKey();
CheckResult result = checker.check(key);
if (!result.isOk()) {
throw new BadRequestException(String.format("Problems with public key %s:\n%s", keyToString(key), Joiner.on('\n').join(result.getProblems())));
}
store.add(keyRing);
}
for (Fingerprint fp : toRemove) {
store.remove(fp.get());
}
CommitBuilder cb = new CommitBuilder();
PersonIdent committer = serverIdent.get();
cb.setAuthor(rsrc.getUser().newCommitterIdent(committer.getWhen(), committer.getTimeZone()));
cb.setCommitter(committer);
RefUpdate.Result saveResult = store.save(cb);
switch(saveResult) {
case NEW:
case FAST_FORWARD:
case FORCED:
case NO_CHANGE:
break;
default:
// TODO(dborowitz): Backoff and retry on LOCK_FAILURE.
throw new ResourceConflictException("Failed to save public keys: " + saveResult);
}
}
This is not exactly the key stored in the store, but is equivalent. In
particular, it will have a Bouncy Castle version string. The armored
stream reader in PublicKeyStore doesn't give us an easy way to extract
the original ASCII armor.
PGPPublicKey key = keyRing.getPublicKey();
GpgKeyInfo info = new GpgKeyInfo();
info.id = PublicKeyStore.keyIdToString(key.getKeyID());
info.fingerprint = PublicKeyStore.fingerprintToString(key.getFingerprint());
@SuppressWarnings("unchecked")
Iterator userIds = key.getUserIDs();
info.userIds = ImmutableList.copyOf(userIds);
try (ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
ArmoredOutputStream aout = new ArmoredOutputStream(out)) {
// This is not exactly the key stored in the store, but is equivalent. In
// particular, it will have a Bouncy Castle version string. The armored
// stream reader in PublicKeyStore doesn't give us an easy way to extract
// the original ASCII armor.
key.encode(aout);
info.key = new String(out.toByteArray(), UTF_8);
}
return info;
try (PublicKeyStore store = storeProvider.get()) {
for (PGPPublicKeyRing keyRing : keyRings) {
PGPPublicKey key = keyRing.getPublicKey();
CheckResult result = checker.check(key);
if (!result.isOk()) {
throw new BadRequestException(String.format("Problems with public key %s:\n%s", keyToString(key), Joiner.on('\n').join(result.getProblems())));
}
store.add(keyRing);
}
CommitBuilder cb = new CommitBuilder();
PersonIdent committer = serverIdent.get();
cb.setAuthor(rsrc.getUser().newCommitterIdent(committer.getWhen(), committer.getTimeZone()));
cb.setCommitter(committer);
RefUpdate.Result saveResult = store.save(cb);
switch(saveResult) {
case NEW:
case FAST_FORWARD:
case FORCED:
break;
default:
// TODO(dborowitz): Backoff and retry on LOCK_FAILURE.
throw new ResourceConflictException("Failed to save public key: " + saveResult);
}
}
If the container didn't do the authentication we might
have done it in the front-end web server. Try to split
the identity out of the Authorization header and honor it.
SATD_ADDED
getRemoteUser(HttpServletRequest, String)
public static String getRemoteUser(HttpServletRequest req, String loginHeader)
if (AUTHORIZATION.equals(loginHeader)) {
String user = emptyToNull(req.getRemoteUser());
if (user != null) {
// The container performed the authentication, and has the user
// identity already decoded for us. Honor that as we have been
// configured to honor HTTP authentication.
return user;
}
// If the container didn't do the authentication we might
// have done it in the front-end web server. Try to split
// the identity out of the Authorization header and honor it.
String auth = req.getHeader(AUTHORIZATION);
return extractUsername(auth);
} else {
// Nonstandard HTTP header. We have been told to trust this
// header blindly as-is.
return emptyToNull(req.getHeader(loginHeader));
}
We sometimes collapsed an edit together in a strange way,
such that the edges of each text is identical. Fix by
by dropping out that incorrectly replaced region.
We sometimes collapsed an edit together in a strange way,
such that the edges of each text is identical. Fix by
by dropping out that incorrectly replaced region.
CLASS_OR_METHOD_CHANGED
compute(Text, Text, List)
static IntraLineDiff compute(Text aText, Text bText, List edits) throws Exception
combineLineEdits(edits, aText, bText);
for (int i = 0; i < edits.size(); i++) {
Edit e = edits.get(i);
if (e.getType() == Edit.Type.REPLACE) {
CharText a = new CharText(aText, e.getBeginA(), e.getEndA());
CharText b = new CharText(bText, e.getBeginB(), e.getEndB());
CharTextComparator cmp = new CharTextComparator();
List wordEdits = MyersDiff.INSTANCE.diff(cmp, a, b);
// Combine edits that are really close together. If they are
// just a few characters apart we tend to get better results
// by joining them together and taking the whole span.
//
for (int j = 0; j < wordEdits.size() - 1; ) {
Edit c = wordEdits.get(j);
Edit n = wordEdits.get(j + 1);
if (n.getBeginA() - c.getEndA() <= 5 || n.getBeginB() - c.getEndB() <= 5) {
int ab = c.getBeginA();
int ae = n.getEndA();
int bb = c.getBeginB();
int be = n.getEndB();
if (canCoalesce(a, c.getEndA(), n.getBeginA()) && canCoalesce(b, c.getEndB(), n.getBeginB())) {
wordEdits.set(j, new Edit(ab, ae, bb, be));
wordEdits.remove(j + 1);
continue;
}
}
j++;
}
// Apply some simple rules to fix up some of the edits. Our
// logic above, along with our per-character difference tends
// to produce some crazy stuff.
//
for (int j = 0; j < wordEdits.size(); j++) {
Edit c = wordEdits.get(j);
int ab = c.getBeginA();
int ae = c.getEndA();
int bb = c.getBeginB();
int be = c.getEndB();
// Sometimes the diff generator produces an INSERT or DELETE
// right up against a REPLACE, but we only find this after
// we've also played some shifting games on the prior edit.
// If that happened to us, coalesce them together so we can
// correct this mess for the user. If we don't we wind up
// with silly stuff like "es" -> "es = Addresses".
//
if (1 < j) {
Edit p = wordEdits.get(j - 1);
if (p.getEndA() == ab || p.getEndB() == bb) {
if (p.getEndA() == ab && p.getBeginA() < p.getEndA()) {
ab = p.getBeginA();
}
if (p.getEndB() == bb && p.getBeginB() < p.getEndB()) {
bb = p.getBeginB();
}
wordEdits.remove(--j);
}
}
// We sometimes collapsed an edit together in a strange way,
// such that the edges of each text is identical. Fix by
// by dropping out that incorrectly replaced region.
//
while (ab < ae && bb < be && cmp.equals(a, ab, b, bb)) {
ab++;
bb++;
}
while (ab < ae && bb < be && cmp.equals(a, ae - 1, b, be - 1)) {
ae--;
be--;
}
// The leading part of an edit and its trailing part in the same
// text might be identical. Slide down that edit and use the tail
// rather than the leading bit. If however the edit is only on a
// whitespace block try to shift it to the left margin, assuming
// that it is an indentation change.
//
boolean aShift = true;
if (ab < ae && isOnlyWhitespace(a, ab, ae)) {
int lf = findLF(wordEdits, j, a, ab);
if (lf < ab && a.charAt(lf) == '\n') {
int nb = lf + 1;
int p = 0;
while (p < ae - ab) {
if (cmp.equals(a, ab + p, a, ab + p)) {
p++;
} else {
break;
}
}
if (p == ae - ab) {
ab = nb;
ae = nb + p;
aShift = false;
}
}
}
if (aShift) {
while (0 < ab && ab < ae && a.charAt(ab - 1) != '\n' && cmp.equals(a, ab - 1, a, ae - 1)) {
ab--;
ae--;
}
if (!a.isLineStart(ab) || !a.contains(ab, ae, '\n')) {
while (ab < ae && ae < a.size() && cmp.equals(a, ab, a, ae)) {
ab++;
ae++;
if (a.charAt(ae - 1) == '\n') {
break;
}
}
}
}
boolean bShift = true;
if (bb < be && isOnlyWhitespace(b, bb, be)) {
int lf = findLF(wordEdits, j, b, bb);
if (lf < bb && b.charAt(lf) == '\n') {
int nb = lf + 1;
int p = 0;
while (p < be - bb) {
if (cmp.equals(b, bb + p, b, bb + p)) {
p++;
} else {
break;
}
}
if (p == be - bb) {
bb = nb;
be = nb + p;
bShift = false;
}
}
}
if (bShift) {
while (0 < bb && bb < be && b.charAt(bb - 1) != '\n' && cmp.equals(b, bb - 1, b, be - 1)) {
bb--;
be--;
}
if (!b.isLineStart(bb) || !b.contains(bb, be, '\n')) {
while (bb < be && be < b.size() && cmp.equals(b, bb, b, be)) {
bb++;
be++;
if (b.charAt(be - 1) == '\n') {
break;
}
}
}
}
// If most of a line was modified except the LF was common, make
// the LF part of the modification region. This is easier to read.
//
if (//
ab < ae && //
(ab == 0 || a.charAt(ab - 1) == '\n') && ae < a.size() && a.charAt(ae - 1) != '\n' && a.charAt(ae) == '\n') {
ae++;
}
if (//
bb < be && //
(bb == 0 || b.charAt(bb - 1) == '\n') && be < b.size() && b.charAt(be - 1) != '\n' && b.charAt(be) == '\n') {
be++;
}
wordEdits.set(j, new Edit(ab, ae, bb, be));
}
edits.set(i, new ReplaceEdit(e, wordEdits));
}
}
return new IntraLineDiff(edits);
Apply some simple rules to fix up some of the edits. Our
logic above, along with our per-character difference tends
to produce some crazy stuff.
Apply some simple rules to fix up some of the edits. Our
logic above, along with our per-character difference tends
to produce some crazy stuff.
CLASS_OR_METHOD_CHANGED
compute(Text, Text, List)
static IntraLineDiff compute(Text aText, Text bText, List edits) throws Exception
combineLineEdits(edits, aText, bText);
for (int i = 0; i < edits.size(); i++) {
Edit e = edits.get(i);
if (e.getType() == Edit.Type.REPLACE) {
CharText a = new CharText(aText, e.getBeginA(), e.getEndA());
CharText b = new CharText(bText, e.getBeginB(), e.getEndB());
CharTextComparator cmp = new CharTextComparator();
List wordEdits = MyersDiff.INSTANCE.diff(cmp, a, b);
// Combine edits that are really close together. If they are
// just a few characters apart we tend to get better results
// by joining them together and taking the whole span.
//
for (int j = 0; j < wordEdits.size() - 1; ) {
Edit c = wordEdits.get(j);
Edit n = wordEdits.get(j + 1);
if (n.getBeginA() - c.getEndA() <= 5 || n.getBeginB() - c.getEndB() <= 5) {
int ab = c.getBeginA();
int ae = n.getEndA();
int bb = c.getBeginB();
int be = n.getEndB();
if (canCoalesce(a, c.getEndA(), n.getBeginA()) && canCoalesce(b, c.getEndB(), n.getBeginB())) {
wordEdits.set(j, new Edit(ab, ae, bb, be));
wordEdits.remove(j + 1);
continue;
}
}
j++;
}
// Apply some simple rules to fix up some of the edits. Our
// logic above, along with our per-character difference tends
// to produce some crazy stuff.
//
for (int j = 0; j < wordEdits.size(); j++) {
Edit c = wordEdits.get(j);
int ab = c.getBeginA();
int ae = c.getEndA();
int bb = c.getBeginB();
int be = c.getEndB();
// Sometimes the diff generator produces an INSERT or DELETE
// right up against a REPLACE, but we only find this after
// we've also played some shifting games on the prior edit.
// If that happened to us, coalesce them together so we can
// correct this mess for the user. If we don't we wind up
// with silly stuff like "es" -> "es = Addresses".
//
if (1 < j) {
Edit p = wordEdits.get(j - 1);
if (p.getEndA() == ab || p.getEndB() == bb) {
if (p.getEndA() == ab && p.getBeginA() < p.getEndA()) {
ab = p.getBeginA();
}
if (p.getEndB() == bb && p.getBeginB() < p.getEndB()) {
bb = p.getBeginB();
}
wordEdits.remove(--j);
}
}
// We sometimes collapsed an edit together in a strange way,
// such that the edges of each text is identical. Fix by
// by dropping out that incorrectly replaced region.
//
while (ab < ae && bb < be && cmp.equals(a, ab, b, bb)) {
ab++;
bb++;
}
while (ab < ae && bb < be && cmp.equals(a, ae - 1, b, be - 1)) {
ae--;
be--;
}
// The leading part of an edit and its trailing part in the same
// text might be identical. Slide down that edit and use the tail
// rather than the leading bit. If however the edit is only on a
// whitespace block try to shift it to the left margin, assuming
// that it is an indentation change.
//
boolean aShift = true;
if (ab < ae && isOnlyWhitespace(a, ab, ae)) {
int lf = findLF(wordEdits, j, a, ab);
if (lf < ab && a.charAt(lf) == '\n') {
int nb = lf + 1;
int p = 0;
while (p < ae - ab) {
if (cmp.equals(a, ab + p, a, ab + p)) {
p++;
} else {
break;
}
}
if (p == ae - ab) {
ab = nb;
ae = nb + p;
aShift = false;
}
}
}
if (aShift) {
while (0 < ab && ab < ae && a.charAt(ab - 1) != '\n' && cmp.equals(a, ab - 1, a, ae - 1)) {
ab--;
ae--;
}
if (!a.isLineStart(ab) || !a.contains(ab, ae, '\n')) {
while (ab < ae && ae < a.size() && cmp.equals(a, ab, a, ae)) {
ab++;
ae++;
if (a.charAt(ae - 1) == '\n') {
break;
}
}
}
}
boolean bShift = true;
if (bb < be && isOnlyWhitespace(b, bb, be)) {
int lf = findLF(wordEdits, j, b, bb);
if (lf < bb && b.charAt(lf) == '\n') {
int nb = lf + 1;
int p = 0;
while (p < be - bb) {
if (cmp.equals(b, bb + p, b, bb + p)) {
p++;
} else {
break;
}
}
if (p == be - bb) {
bb = nb;
be = nb + p;
bShift = false;
}
}
}
if (bShift) {
while (0 < bb && bb < be && b.charAt(bb - 1) != '\n' && cmp.equals(b, bb - 1, b, be - 1)) {
bb--;
be--;
}
if (!b.isLineStart(bb) || !b.contains(bb, be, '\n')) {
while (bb < be && be < b.size() && cmp.equals(b, bb, b, be)) {
bb++;
be++;
if (b.charAt(be - 1) == '\n') {
break;
}
}
}
}
// If most of a line was modified except the LF was common, make
// the LF part of the modification region. This is easier to read.
//
if (//
ab < ae && //
(ab == 0 || a.charAt(ab - 1) == '\n') && ae < a.size() && a.charAt(ae - 1) != '\n' && a.charAt(ae) == '\n') {
ae++;
}
if (//
bb < be && //
(bb == 0 || b.charAt(bb - 1) == '\n') && be < b.size() && b.charAt(be - 1) != '\n' && b.charAt(be) == '\n') {
be++;
}
wordEdits.set(j, new Edit(ab, ae, bb, be));
}
edits.set(i, new ReplaceEdit(e, wordEdits));
}
}
return new IntraLineDiff(edits);
Combine edits that are really close together. If they are
just a few characters apart we tend to get better results
by joining them together and taking the whole span.
Combine edits that are really close together. If they are
just a few characters apart we tend to get better results
by joining them together and taking the whole span.
CLASS_OR_METHOD_CHANGED
compute(Text, Text, List)
static IntraLineDiff compute(Text aText, Text bText, List edits) throws Exception
combineLineEdits(edits, aText, bText);
for (int i = 0; i < edits.size(); i++) {
Edit e = edits.get(i);
if (e.getType() == Edit.Type.REPLACE) {
CharText a = new CharText(aText, e.getBeginA(), e.getEndA());
CharText b = new CharText(bText, e.getBeginB(), e.getEndB());
CharTextComparator cmp = new CharTextComparator();
List wordEdits = MyersDiff.INSTANCE.diff(cmp, a, b);
// Combine edits that are really close together. If they are
// just a few characters apart we tend to get better results
// by joining them together and taking the whole span.
//
for (int j = 0; j < wordEdits.size() - 1; ) {
Edit c = wordEdits.get(j);
Edit n = wordEdits.get(j + 1);
if (n.getBeginA() - c.getEndA() <= 5 || n.getBeginB() - c.getEndB() <= 5) {
int ab = c.getBeginA();
int ae = n.getEndA();
int bb = c.getBeginB();
int be = n.getEndB();
if (canCoalesce(a, c.getEndA(), n.getBeginA()) && canCoalesce(b, c.getEndB(), n.getBeginB())) {
wordEdits.set(j, new Edit(ab, ae, bb, be));
wordEdits.remove(j + 1);
continue;
}
}
j++;
}
// Apply some simple rules to fix up some of the edits. Our
// logic above, along with our per-character difference tends
// to produce some crazy stuff.
//
for (int j = 0; j < wordEdits.size(); j++) {
Edit c = wordEdits.get(j);
int ab = c.getBeginA();
int ae = c.getEndA();
int bb = c.getBeginB();
int be = c.getEndB();
// Sometimes the diff generator produces an INSERT or DELETE
// right up against a REPLACE, but we only find this after
// we've also played some shifting games on the prior edit.
// If that happened to us, coalesce them together so we can
// correct this mess for the user. If we don't we wind up
// with silly stuff like "es" -> "es = Addresses".
//
if (1 < j) {
Edit p = wordEdits.get(j - 1);
if (p.getEndA() == ab || p.getEndB() == bb) {
if (p.getEndA() == ab && p.getBeginA() < p.getEndA()) {
ab = p.getBeginA();
}
if (p.getEndB() == bb && p.getBeginB() < p.getEndB()) {
bb = p.getBeginB();
}
wordEdits.remove(--j);
}
}
// We sometimes collapsed an edit together in a strange way,
// such that the edges of each text is identical. Fix by
// by dropping out that incorrectly replaced region.
//
while (ab < ae && bb < be && cmp.equals(a, ab, b, bb)) {
ab++;
bb++;
}
while (ab < ae && bb < be && cmp.equals(a, ae - 1, b, be - 1)) {
ae--;
be--;
}
// The leading part of an edit and its trailing part in the same
// text might be identical. Slide down that edit and use the tail
// rather than the leading bit. If however the edit is only on a
// whitespace block try to shift it to the left margin, assuming
// that it is an indentation change.
//
boolean aShift = true;
if (ab < ae && isOnlyWhitespace(a, ab, ae)) {
int lf = findLF(wordEdits, j, a, ab);
if (lf < ab && a.charAt(lf) == '\n') {
int nb = lf + 1;
int p = 0;
while (p < ae - ab) {
if (cmp.equals(a, ab + p, a, ab + p)) {
p++;
} else {
break;
}
}
if (p == ae - ab) {
ab = nb;
ae = nb + p;
aShift = false;
}
}
}
if (aShift) {
while (0 < ab && ab < ae && a.charAt(ab - 1) != '\n' && cmp.equals(a, ab - 1, a, ae - 1)) {
ab--;
ae--;
}
if (!a.isLineStart(ab) || !a.contains(ab, ae, '\n')) {
while (ab < ae && ae < a.size() && cmp.equals(a, ab, a, ae)) {
ab++;
ae++;
if (a.charAt(ae - 1) == '\n') {
break;
}
}
}
}
boolean bShift = true;
if (bb < be && isOnlyWhitespace(b, bb, be)) {
int lf = findLF(wordEdits, j, b, bb);
if (lf < bb && b.charAt(lf) == '\n') {
int nb = lf + 1;
int p = 0;
while (p < be - bb) {
if (cmp.equals(b, bb + p, b, bb + p)) {
p++;
} else {
break;
}
}
if (p == be - bb) {
bb = nb;
be = nb + p;
bShift = false;
}
}
}
if (bShift) {
while (0 < bb && bb < be && b.charAt(bb - 1) != '\n' && cmp.equals(b, bb - 1, b, be - 1)) {
bb--;
be--;
}
if (!b.isLineStart(bb) || !b.contains(bb, be, '\n')) {
while (bb < be && be < b.size() && cmp.equals(b, bb, b, be)) {
bb++;
be++;
if (b.charAt(be - 1) == '\n') {
break;
}
}
}
}
// If most of a line was modified except the LF was common, make
// the LF part of the modification region. This is easier to read.
//
if (//
ab < ae && //
(ab == 0 || a.charAt(ab - 1) == '\n') && ae < a.size() && a.charAt(ae - 1) != '\n' && a.charAt(ae) == '\n') {
ae++;
}
if (//
bb < be && //
(bb == 0 || b.charAt(bb - 1) == '\n') && be < b.size() && b.charAt(be - 1) != '\n' && b.charAt(be) == '\n') {
be++;
}
wordEdits.set(j, new Edit(ab, ae, bb, be));
}
edits.set(i, new ReplaceEdit(e, wordEdits));
}
}
return new IntraLineDiff(edits);
TODO(sbeller): why do we need to treat followup specially here?
The followup action is a client-side only operation that does not
have a server side handler. It must be manually registered into the
resulting action map.
SATD_REMOVED
ActionInfo> toActionMap(ChangeControl)
private Map toActionMap(ChangeControl ctl)
Map out = new LinkedHashMap<>();
if (!ctl.getCurrentUser().isIdentifiedUser()) {
return out;
}
Provider userProvider = Providers.of(ctl.getCurrentUser());
for (UiAction.Description d : UiActions.from(changeViews, new ChangeResource(ctl), userProvider)) {
out.put(d.getId(), new ActionInfo(d));
}
// The followup action is a client-side only operation that does not
// have a server side handler. It must be manually registered into the
// resulting action map.
if (ctl.getChange().getStatus().isOpen()) {
UiAction.Description descr = new UiAction.Description();
PrivateInternals_UiActionDescription.setId(descr, "followup");
PrivateInternals_UiActionDescription.setMethod(descr, "POST");
descr.setTitle("Create follow-up change");
out.put(descr.getId(), new ActionInfo(descr));
}
return out;
logDebug("Validating {} changes", submitted.size());
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new MergeException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new MergeException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.SUBMITTED && chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new or submitted: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new MergeException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = (CodeReviewCommit) rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
continue;
}
commit.add(canMergeFlag);
toSubmit.put(submitType, cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return toSubmit;
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
logDebug("Validating {} changes", submitted.size());
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new MergeException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
// Reload change in case index was stale.
chg = cd.reloadChange();
} catch (OrmException e) {
throw new MergeException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.getStatus() != Change.Status.SUBMITTED && chg.getStatus() != Change.Status.NEW) {
logDebug("Change {} is not new or submitted: {}", changeId, chg.getStatus());
continue;
}
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
PatchSet ps;
Branch.NameKey destBranch = chg.getDest();
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new MergeException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
CodeReviewCommit commit;
try {
commit = (CodeReviewCommit) rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
continue;
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
continue;
}
commit.add(canMergeFlag);
toSubmit.put(submitType, cd);
}
logDebug("Submitting on this run: {}", toSubmit);
return toSubmit;
The identity information reported about the connection by a
RFC 1413 [11] request to the remote agent, if
available. Servers MAY choose not to support this feature, or
not to request the data for efficiency reasons.
\"REMOTE_IDENT\" => \"NYI\"
The identity information reported about the connection by a
RFC 1413 [11] request to the remote agent, if
available. Servers MAY choose not to support this feature, or
not to request the data for efficiency reasons.
\"REMOTE_IDENT\" => \"NYI\"
The identity information reported about the connection by a
RFC 1413 [11] request to the remote agent, if
available. Servers MAY choose not to support this feature, or
not to request the data for efficiency reasons.
\"REMOTE_IDENT\" => \"NYI\"
The identity information reported about the connection by a
RFC 1413 [11] request to the remote agent, if
available. Servers MAY choose not to support this feature, or
not to request the data for efficiency reasons.
\"REMOTE_IDENT\" => \"NYI\"
if (schema == null) {
return ChangeField.LEGACY_TOPIC2;
}
if (schema.hasField(EXACT_TOPIC)) {
return schema.getFields().get(EXACT_TOPIC.getName());
}
if (schema.hasField(ChangeField.LEGACY_TOPIC2)) {
return schema.getFields().get(ChangeField.LEGACY_TOPIC2.getName());
}
// Not exact, but we cannot do any better.
return schema.getFields().get(ChangeField.LEGACY_TOPIC3.getName());
Only one parent is new in this push. If it is the only parent, just use
that parent's group. If there are multiple parents, perhaps this commit
is a merge of a side branch. This commit belongs in that parent's group
in that case.
SATD_ADDED
visit(RevCommit)
void visit(RevCommit c)
checkState(!done, "visit() called after getGroups()");
Set interestingParents = getInterestingParents(c);
if (interestingParents.size() == 0) {
// All parents are uninteresting: treat this commit as the root of a new
// group of related changes.
groups.put(c, c.name());
return;
} else if (interestingParents.size() == 1) {
// Only one parent is new in this push. If it is the only parent, just use
// that parent's group. If there are multiple parents, perhaps this commit
// is a merge of a side branch. This commit belongs in that parent's group
// in that case.
groups.putAll(c, groups.get(interestingParents.iterator().next()));
return;
}
// Multiple parents, merging at least two branches containing new commits in
// this push.
Set thisCommitGroups = new TreeSet<>();
Set parentGroupsNewInThisPush = Sets.newLinkedHashSetWithExpectedSize(interestingParents.size());
for (RevCommit p : interestingParents) {
Collection parentGroups = groups.get(p);
if (parentGroups.isEmpty()) {
throw new IllegalStateException(String.format("no group assigned to parent %s of commit %s", p.name(), c.name()));
}
for (String parentGroup : parentGroups) {
if (isGroupFromExistingPatchSet(p, parentGroup)) {
// This parent's group is from an existing patch set, i.e. the parent
// not new in this push. Use this group for the commit.
thisCommitGroups.add(parentGroup);
} else {
// This parent's group is new in this push.
parentGroupsNewInThisPush.add(parentGroup);
}
}
}
Iterable toAlias;
if (thisCommitGroups.isEmpty()) {
// All parent groups were new in this push. Pick the first one and alias
// other parents' groups to this first parent.
String firstParentGroup = parentGroupsNewInThisPush.iterator().next();
thisCommitGroups = ImmutableSet.of(firstParentGroup);
toAlias = Iterables.skip(parentGroupsNewInThisPush, 1);
} else {
// For each parent group that was new in this push, alias it to the actual
// computed group(s) for this commit.
toAlias = parentGroupsNewInThisPush;
}
groups.putAll(c, thisCommitGroups);
for (String pg : toAlias) {
groupAliases.putAll(pg, thisCommitGroups);
}
If the container didn't do the authentication we might
have done it in the front-end web server. Try to split
the identity out of the Authorization header and honor it.
SATD_ADDED
getRemoteUser(HttpServletRequest, String)
public static String getRemoteUser(HttpServletRequest req, String loginHeader)
if (AUTHORIZATION.equals(loginHeader)) {
String user = emptyToNull(req.getRemoteUser());
if (user != null) {
// The container performed the authentication, and has the user
// identity already decoded for us. Honor that as we have been
// configured to honor HTTP authentication.
return user;
}
// If the container didn't do the authentication we might
// have done it in the front-end web server. Try to split
// the identity out of the Authorization header and honor it.
String auth = req.getHeader(AUTHORIZATION);
return extractUsername(auth);
} else {
// Nonstandard HTTP header. We have been told to trust this
// header blindly as-is.
return emptyToNull(req.getHeader(loginHeader));
}
StringBuilder s = new StringBuilder();
JsArray c = content();
for (int i = 0; i < c.length(); i++) {
Region r = c.get(i);
if (r.ab() != null) {
append(s, r.ab());
} else if (r.a() != null) {
append(s, r.a());
}
// TODO skip may need to be handled
}
return s.toString();
StringBuilder s = new StringBuilder();
JsArray c = content();
for (int i = 0; i < c.length(); i++) {
Region r = c.get(i);
if (r.ab() != null) {
append(s, r.ab());
} else if (r.a() != null) {
append(s, r.a());
}
// TODO skip may need to be handled
}
return s.toString();
final List>> r = new ArrayList<>(1);
// TODO(dborowitz): Could eliminate this call by adding an option to include
// inline comments in the change detail.
ChangeApi.comments(changeId.get()).get(group.add(new AsyncCallback>>() {
@Override
public void onSuccess(NativeMap> result) {
// Return value is used for populating the file table, so only count
// comments for the current revision. Still include all comments in
// the history table.
r.add(filterForRevision(result, rev._number()));
history.addComments(result);
}
@Override
public void onFailure(Throwable caught) {
}
}));
return r;
TODO(dborowitz): Merge this with AcceptanceTestRequestScope.
SATD_ADDED
module()
static Module module()
return new AbstractModule() {
@Override
public void configure() {
install(new GerritRequestModule());
bind(RequestScopePropagator.class).to(Propagator.class);
bindScope(RequestScoped.class, InProcessProtocol.REQUEST);
}
@Provides
@RemotePeer
SocketAddress getSocketAddress() {
// TODO(dborowitz): Could potentially fake this with thread ID or
// something.
throw new OutOfScopeException("No remote peer in acceptance tests");
}
};
no empty paths
no suffix
no windows/dos style paths
no absolute paths
no absolute paths
no \"l../etc/passwd\"
no \"foo/../etc/passwd\"
\"foo/./foo\" is insane to ask
windows UNC path can be \"//...\"
common unix wildcard
wildcard or string parameter
wildcard
Could be used for absolute paths in windows?
redirect input
redirect output
pipe
dollar sign
carriage return
The results of this method are cached by ProjectCacheImpl. Control only
enters here if the cache was flushed by the administrator to force
scanning the filesystem. Don't rely on the cached names collection.
no empty paths
no suffix
no windows/dos style paths
no absolute paths
no absolute paths
no \"l../etc/passwd\"
no \"foo/../etc/passwd\"
\"foo/./foo\" is insane to ask
windows UNC path can be \"//...\"
common unix wildcard
wildcard or string parameter
wildcard
Could be used for absolute paths in windows?
redirect input
redirect output
pipe
dollar sign
carriage return
no empty paths
no suffix
no windows/dos style paths
no absolute paths
no absolute paths
no \"l../etc/passwd\"
no \"foo/../etc/passwd\"
\"foo/./foo\" is insane to ask
windows UNC path can be \"//...\"
no path segments that end with '.git' as \"foo.git/bar\"
common unix wildcard
wildcard or string parameter
wildcard
Could be used for absolute paths in windows?
redirect input
redirect output
pipe
dollar sign
carriage return
private void doOnlySubscriptionTableOperations(final String gitModulesFileContent, final Branch.NameKey mergedBranch, final List extractedSubscriptions, final List previousSubscriptions) throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository realDb = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git git = new Git(realDb);
addRegularFileToIndex(".gitmodules", gitModulesFileContent, realDb);
final RevCommit mergeTip = git.commit().setMessage("test").call();
expect(urlProvider.get()).andReturn("http://localhost:8080").times(2);
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
expect(subscriptions.bySuperProject(mergedBranch)).andReturn(new ListResultSet<>(previousSubscriptions));
SortedSet existingProjects = new TreeSet<>();
for (SubmoduleSubscription extracted : extractedSubscriptions) {
existingProjects.add(extracted.getSubmodule().getParentKey());
}
for (int index = 0; index < extractedSubscriptions.size(); index++) {
expect(repoManager.list()).andReturn(existingProjects);
}
final Set alreadySubscribeds = new HashSet<>();
for (SubmoduleSubscription s : extractedSubscriptions) {
if (previousSubscriptions.contains(s)) {
alreadySubscribeds.add(s);
}
}
final Set subscriptionsToRemove = new HashSet<>(previousSubscriptions);
final List subscriptionsToInsert = new ArrayList<>(extractedSubscriptions);
subscriptionsToRemove.removeAll(subscriptionsToInsert);
subscriptionsToInsert.removeAll(alreadySubscribeds);
if (!subscriptionsToRemove.isEmpty()) {
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
subscriptions.delete(subscriptionsToRemove);
}
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
subscriptions.insert(subscriptionsToInsert);
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
expect(subscriptions.bySubmodule(mergedBranch)).andReturn(new ListResultSet<>(new ArrayList()));
schema.close();
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(mergedBranch, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, new Project(mergedBranch.getParentKey()), new ArrayList(), null, null, repoManager, null, null, null);
submoduleOp.update();
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
testAvoidingCircularReference()
public void testAvoidingCircularReference() throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository sourceRepository = createWorkRepository();
Repository targetRepository = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git sourceGit = new Git(sourceRepository);
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git targetGit = new Git(targetRepository);
addRegularFileToIndex("file.txt", "test content", sourceRepository);
final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call();
final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master");
final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId());
final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs());
final Map mergedCommits = new HashMap<>();
mergedCommits.put(submittedChange.getId(), codeReviewCommit);
final List submitted = new ArrayList<>();
submitted.add(submittedChange);
addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository);
targetGit.commit().setMessage("test").call();
final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get());
expect(urlProvider.get()).andReturn("http://localhost:8080");
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project")));
expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers);
expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes();
Capture ruCapture = new Capture<>();
gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture));
changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull());
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet incorrectSubscriptions = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(sourceBranchNameKey, targetBranchNameKey, "target-project")));
expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(incorrectSubscriptions);
schema.close();
final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com");
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks);
submoduleOp.update();
doVerify();
RefUpdate ru = ruCapture.getValue();
assertEquals(ru.getName(), targetBranchNameKey.get());
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
testOneSubscriberToUpdate()
public void testOneSubscriberToUpdate() throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository sourceRepository = createWorkRepository();
Repository targetRepository = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git sourceGit = new Git(sourceRepository);
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git targetGit = new Git(targetRepository);
addRegularFileToIndex("file.txt", "test content", sourceRepository);
final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call();
final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master");
final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId());
final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs());
final Map mergedCommits = new HashMap<>();
mergedCommits.put(submittedChange.getId(), codeReviewCommit);
final List submitted = new ArrayList<>();
submitted.add(submittedChange);
addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository);
targetGit.commit().setMessage("test").call();
final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get());
expect(urlProvider.get()).andReturn("http://localhost:8080");
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project")));
expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers);
expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes();
Capture ruCapture = new Capture<>();
gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture));
changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull());
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList());
expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(emptySubscriptions);
schema.close();
final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com");
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks);
submoduleOp.update();
doVerify();
RefUpdate ru = ruCapture.getValue();
assertEquals(ru.getName(), targetBranchNameKey.get());
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
testEmptyCommit()
public void testEmptyCommit() throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository realDb = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git git = new Git(realDb);
final RevCommit mergeTip = git.commit().setMessage("test").call();
final Branch.NameKey branchNameKey = new Branch.NameKey(new Project.NameKey("test-project"), "test-branch");
expect(urlProvider.get()).andReturn("http://localhost:8080");
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList());
expect(subscriptions.bySubmodule(branchNameKey)).andReturn(emptySubscriptions);
schema.close();
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(branchNameKey, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, null, new ArrayList(), null, null, null, null, null, null);
submoduleOp.update();
doVerify();
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
setUp()
public void setUp() throws Exception
super.setUp();
/*- The following graph will be created.
o tag 2.5, 2.5_annotated, 2.5_annotated_twice
|\
| o tag 2.0.1
| o tag 2.0
o | tag 1.3
|/
o c3
| o tag 1.0.1
|/
o tag 1.0
o c2
o c1
*/
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
Git git = new Git(db);
revWalk = new RevWalk(db);
// Version 1.0
commit_initial = git.commit().setMessage("c1").call();
git.commit().setMessage("c2").call();
RevCommit commit_v1_0 = git.commit().setMessage("version 1.0").call();
git.tag().setName(TAG_1_0).setObjectId(commit_v1_0).call();
RevCommit c3 = git.commit().setMessage("c3").call();
// Version 1.01
createAndCheckoutBranch(commit_v1_0, BRANCH_1_0);
RevCommit commit_v1_0_1 = git.commit().setMessage("verREFS_HEADS_RELsion 1.0.1").call();
git.tag().setName(TAG_1_0_1).setObjectId(commit_v1_0_1).call();
// Version 1.3
createAndCheckoutBranch(c3, BRANCH_1_3);
commit_v1_3 = git.commit().setMessage("version 1.3").call();
git.tag().setName(TAG_1_3).setObjectId(commit_v1_3).call();
// Version 2.0
createAndCheckoutBranch(c3, BRANCH_2_0);
RevCommit commit_v2_0 = git.commit().setMessage("version 2.0").call();
git.tag().setName(TAG_2_0).setObjectId(commit_v2_0).call();
RevCommit commit_v2_0_1 = git.commit().setMessage("version 2.0.1").call();
git.tag().setName(TAG_2_0_1).setObjectId(commit_v2_0_1).call();
// Version 2.5
createAndCheckoutBranch(commit_v1_3, BRANCH_2_5);
git.merge().include(commit_v2_0_1).setCommit(false).setFastForward(FastForwardMode.NO_FF).call();
commit_v2_5 = git.commit().setMessage("version 2.5").call();
git.tag().setName(TAG_2_5).setObjectId(commit_v2_5).setAnnotated(false).call();
Ref ref_tag_2_5_annotated = git.tag().setName(TAG_2_5_ANNOTATED).setObjectId(commit_v2_5).setAnnotated(true).call();
RevTag tag_2_5_annotated = revWalk.parseTag(ref_tag_2_5_annotated.getObjectId());
git.tag().setName(TAG_2_5_ANNOTATED_TWICE).setObjectId(tag_2_5_annotated).setAnnotated(true).call();
no \"l../etc/passwd\"
no \"foo/../etc/passwd\"
\"foo/./foo\" is insane to ask
windows UNC path can be \"//...\"
common unix wildcard
wildcard or string parameter
wildcard
Could be used for absolute paths in windows?
redirect input
redirect output
pipe
dollar sign
carriage return
The results of this method are cached by ProjectCacheImpl. Control only
enters here if the cache was flushed by the administrator to force
scanning the filesystem. Don't rely on the cached names collection.
no \"l../etc/passwd\"
no \"foo/../etc/passwd\"
\"foo/./foo\" is insane to ask
windows UNC path can be \"//...\"
common unix wildcard
wildcard or string parameter
wildcard
Could be used for absolute paths in windows?
redirect input
redirect output
pipe
dollar sign
carriage return
no empty paths
no suffix
no windows/dos style paths
no absolute paths
no absolute paths
no \"l../etc/passwd\"
no \"foo/../etc/passwd\"
\"foo/./foo\" is insane to ask
windows UNC path can be \"//...\"
common unix wildcard
wildcard or string parameter
wildcard
Could be used for absolute paths in windows?
redirect input
redirect output
pipe
dollar sign
carriage return
For a file with content merge conflict that we produced a result
above on, collapse the file down to a single stage 0 with just
the blob content, and a randomly selected mode (the lowest stage,
which should be the merge base, or ours).
For a file with content merge conflict that we produced a result
above on, collapse the file down to a single stage 0 with just
the blob content, and a randomly selected mode (the lowest stage,
which should be the merge base, or ours).
String hash = b.name();
String refName = RefNames.REFS_CACHE_AUTOMERGE + hash.substring(0, 2) + "/" + hash.substring(2);
Ref ref = repo.getRef(refName);
if (ref != null && ref.getObjectId() != null) {
return rw.parseTree(ref.getObjectId());
}
ResolveMerger m = (ResolveMerger) mergeStrategy.newMerger(repo, true);
final ObjectInserter ins = repo.newObjectInserter();
try {
DirCache dc = DirCache.newInCore();
m.setDirCache(dc);
m.setObjectInserter(new ObjectInserter.Filter() {
@Override
protected ObjectInserter delegate() {
return ins;
}
@Override
public void flush() {
}
@Override
public void release() {
}
});
boolean couldMerge;
try {
couldMerge = m.merge(b.getParents());
} catch (IOException e) {
// It is not safe to continue further down in this method as throwing
// an exception most likely means that the merge tree was not created
// and m.getMergeResults() is empty. This would mean that all paths are
// unmerged and Gerrit UI would show all paths in the patch list.
log.warn("Error attempting automerge " + refName, e);
return null;
}
ObjectId treeId;
if (couldMerge) {
treeId = m.getResultTreeId();
} else {
RevCommit ours = b.getParent(0);
RevCommit theirs = b.getParent(1);
rw.parseBody(ours);
rw.parseBody(theirs);
String oursMsg = ours.getShortMessage();
String theirsMsg = theirs.getShortMessage();
String oursName = String.format("HEAD (%s %s)", ours.abbreviate(6).name(), oursMsg.substring(0, Math.min(oursMsg.length(), 60)));
String theirsName = String.format("BRANCH (%s %s)", theirs.abbreviate(6).name(), theirsMsg.substring(0, Math.min(theirsMsg.length(), 60)));
MergeFormatter fmt = new MergeFormatter();
Map> r = m.getMergeResults();
Map resolved = new HashMap<>();
for (Map.Entry> entry : r.entrySet()) {
MergeResult p = entry.getValue();
TemporaryBuffer buf = new TemporaryBuffer.LocalFile(10 * 1024 * 1024);
try {
fmt.formatMerge(buf, p, "BASE", oursName, theirsName, "UTF-8");
buf.close();
InputStream in = buf.openInputStream();
try {
resolved.put(entry.getKey(), ins.insert(Constants.OBJ_BLOB, buf.length(), in));
} finally {
in.close();
}
} finally {
buf.destroy();
}
}
DirCacheBuilder builder = dc.builder();
int cnt = dc.getEntryCount();
for (int i = 0; i < cnt; ) {
DirCacheEntry entry = dc.getEntry(i);
if (entry.getStage() == 0) {
builder.add(entry);
i++;
continue;
}
int next = dc.nextEntry(i);
String path = entry.getPathString();
DirCacheEntry res = new DirCacheEntry(path);
if (resolved.containsKey(path)) {
// For a file with content merge conflict that we produced a result
// above on, collapse the file down to a single stage 0 with just
// the blob content, and a randomly selected mode (the lowest stage,
// which should be the merge base, or ours).
res.setFileMode(entry.getFileMode());
res.setObjectId(resolved.get(path));
} else if (next == i + 1) {
// If there is exactly one stage present, shouldn't be a conflict...
res.setFileMode(entry.getFileMode());
res.setObjectId(entry.getObjectId());
} else if (next == i + 2) {
// Two stages suggests a delete/modify conflict. Pick the higher
// stage as the automatic result.
entry = dc.getEntry(i + 1);
res.setFileMode(entry.getFileMode());
res.setObjectId(entry.getObjectId());
} else {
// 3 stage conflict, no resolve above
// Punt on the 3-stage conflict and show the base, for now.
res.setFileMode(entry.getFileMode());
res.setObjectId(entry.getObjectId());
}
builder.add(res);
i = next;
}
builder.finish();
treeId = dc.writeTree(ins);
}
ins.flush();
if (save) {
RefUpdate update = repo.updateRef(refName);
update.setNewObjectId(treeId);
update.disableRefLog();
update.forceUpdate();
}
return rw.lookupTree(treeId);
} finally {
ins.release();
}
private void doOnlySubscriptionTableOperations(final String gitModulesFileContent, final Branch.NameKey mergedBranch, final List extractedSubscriptions, final List previousSubscriptions) throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository realDb = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git git = new Git(realDb);
addRegularFileToIndex(".gitmodules", gitModulesFileContent, realDb);
final RevCommit mergeTip = git.commit().setMessage("test").call();
expect(urlProvider.get()).andReturn("http://localhost:8080").times(2);
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
expect(subscriptions.bySuperProject(mergedBranch)).andReturn(new ListResultSet<>(previousSubscriptions));
SortedSet existingProjects = new TreeSet<>();
for (SubmoduleSubscription extracted : extractedSubscriptions) {
existingProjects.add(extracted.getSubmodule().getParentKey());
}
for (int index = 0; index < extractedSubscriptions.size(); index++) {
expect(repoManager.list()).andReturn(existingProjects);
}
final Set alreadySubscribeds = new HashSet<>();
for (SubmoduleSubscription s : extractedSubscriptions) {
if (previousSubscriptions.contains(s)) {
alreadySubscribeds.add(s);
}
}
final Set subscriptionsToRemove = new HashSet<>(previousSubscriptions);
final List subscriptionsToInsert = new ArrayList<>(extractedSubscriptions);
subscriptionsToRemove.removeAll(subscriptionsToInsert);
subscriptionsToInsert.removeAll(alreadySubscribeds);
if (!subscriptionsToRemove.isEmpty()) {
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
subscriptions.delete(subscriptionsToRemove);
}
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
subscriptions.insert(subscriptionsToInsert);
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
expect(subscriptions.bySubmodule(mergedBranch)).andReturn(new ListResultSet<>(new ArrayList()));
schema.close();
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(mergedBranch, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, new Project(mergedBranch.getParentKey()), new ArrayList(), null, null, repoManager, null, null, null);
submoduleOp.update();
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
testAvoidingCircularReference()
public void testAvoidingCircularReference() throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository sourceRepository = createWorkRepository();
Repository targetRepository = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git sourceGit = new Git(sourceRepository);
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git targetGit = new Git(targetRepository);
addRegularFileToIndex("file.txt", "test content", sourceRepository);
final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call();
final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master");
final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId());
final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs());
final Map mergedCommits = new HashMap<>();
mergedCommits.put(submittedChange.getId(), codeReviewCommit);
final List submitted = new ArrayList<>();
submitted.add(submittedChange);
addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository);
targetGit.commit().setMessage("test").call();
final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get());
expect(urlProvider.get()).andReturn("http://localhost:8080");
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project")));
expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers);
expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes();
Capture ruCapture = new Capture<>();
gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture));
changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull());
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet incorrectSubscriptions = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(sourceBranchNameKey, targetBranchNameKey, "target-project")));
expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(incorrectSubscriptions);
schema.close();
final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com");
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks);
submoduleOp.update();
doVerify();
RefUpdate ru = ruCapture.getValue();
assertEquals(ru.getName(), targetBranchNameKey.get());
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
testOneSubscriberToUpdate()
public void testOneSubscriberToUpdate() throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository sourceRepository = createWorkRepository();
Repository targetRepository = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git sourceGit = new Git(sourceRepository);
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git targetGit = new Git(targetRepository);
addRegularFileToIndex("file.txt", "test content", sourceRepository);
final RevCommit sourceMergeTip = sourceGit.commit().setMessage("test").call();
final Branch.NameKey sourceBranchNameKey = new Branch.NameKey(new Project.NameKey("source-project"), "refs/heads/master");
final CodeReviewCommit codeReviewCommit = new CodeReviewCommit(sourceMergeTip.toObjectId());
final Change submittedChange = new Change(new Change.Key(sourceMergeTip.toObjectId().getName()), new Change.Id(1), new Account.Id(1), sourceBranchNameKey, TimeUtil.nowTs());
final Map mergedCommits = new HashMap<>();
mergedCommits.put(submittedChange.getId(), codeReviewCommit);
final List submitted = new ArrayList<>();
submitted.add(submittedChange);
addGitLinkToIndex("a", sourceMergeTip.copy(), targetRepository);
targetGit.commit().setMessage("test").call();
final Branch.NameKey targetBranchNameKey = new Branch.NameKey(new Project.NameKey("target-project"), sourceBranchNameKey.get());
expect(urlProvider.get()).andReturn("http://localhost:8080");
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet subscribers = new ListResultSet<>(Collections.singletonList(new SubmoduleSubscription(targetBranchNameKey, sourceBranchNameKey, "source-project")));
expect(subscriptions.bySubmodule(sourceBranchNameKey)).andReturn(subscribers);
expect(repoManager.openRepository(targetBranchNameKey.getParentKey())).andReturn(targetRepository).anyTimes();
Capture ruCapture = new Capture<>();
gitRefUpdated.fire(eq(targetBranchNameKey.getParentKey()), capture(ruCapture));
changeHooks.doRefUpdatedHook(eq(targetBranchNameKey), anyObject(RefUpdate.class), EasyMock.isNull());
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList());
expect(subscriptions.bySubmodule(targetBranchNameKey)).andReturn(emptySubscriptions);
schema.close();
final PersonIdent myIdent = new PersonIdent("test-user", "test-user@email.com");
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(sourceBranchNameKey, sourceMergeTip, new RevWalk(sourceRepository), urlProvider, schemaFactory, sourceRepository, new Project(sourceBranchNameKey.getParentKey()), submitted, mergedCommits, myIdent, repoManager, gitRefUpdated, null, changeHooks);
submoduleOp.update();
doVerify();
RefUpdate ru = ruCapture.getValue();
assertEquals(ru.getName(), targetBranchNameKey.get());
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
testEmptyCommit()
public void testEmptyCommit() throws Exception
expect(schemaFactory.open()).andReturn(schema);
try (Repository realDb = createWorkRepository()) {
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
final Git git = new Git(realDb);
final RevCommit mergeTip = git.commit().setMessage("test").call();
final Branch.NameKey branchNameKey = new Branch.NameKey(new Project.NameKey("test-project"), "test-branch");
expect(urlProvider.get()).andReturn("http://localhost:8080");
expect(schema.submoduleSubscriptions()).andReturn(subscriptions);
final ResultSet emptySubscriptions = new ListResultSet<>(new ArrayList());
expect(subscriptions.bySubmodule(branchNameKey)).andReturn(emptySubscriptions);
schema.close();
doReplay();
final SubmoduleOp submoduleOp = new SubmoduleOp(branchNameKey, mergeTip, new RevWalk(realDb), urlProvider, schemaFactory, realDb, null, new ArrayList(), null, null, null, null, null, null);
submoduleOp.update();
doVerify();
}
TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
SATD_ADDED
setUp()
public void setUp() throws Exception
super.setUp();
/*- The following graph will be created.
o tag 2.5, 2.5_annotated, 2.5_annotated_twice
|\
| o tag 2.0.1
| o tag 2.0
o | tag 1.3
|/
o c3
| o tag 1.0.1
|/
o tag 1.0
o c2
o c1
*/
// TODO(dborowitz): Use try/finally when this doesn't double-close the repo.
@SuppressWarnings("resource")
Git git = new Git(db);
revWalk = new RevWalk(db);
// Version 1.0
commit_initial = git.commit().setMessage("c1").call();
git.commit().setMessage("c2").call();
RevCommit commit_v1_0 = git.commit().setMessage("version 1.0").call();
git.tag().setName(TAG_1_0).setObjectId(commit_v1_0).call();
RevCommit c3 = git.commit().setMessage("c3").call();
// Version 1.01
createAndCheckoutBranch(commit_v1_0, BRANCH_1_0);
RevCommit commit_v1_0_1 = git.commit().setMessage("verREFS_HEADS_RELsion 1.0.1").call();
git.tag().setName(TAG_1_0_1).setObjectId(commit_v1_0_1).call();
// Version 1.3
createAndCheckoutBranch(c3, BRANCH_1_3);
commit_v1_3 = git.commit().setMessage("version 1.3").call();
git.tag().setName(TAG_1_3).setObjectId(commit_v1_3).call();
// Version 2.0
createAndCheckoutBranch(c3, BRANCH_2_0);
RevCommit commit_v2_0 = git.commit().setMessage("version 2.0").call();
git.tag().setName(TAG_2_0).setObjectId(commit_v2_0).call();
RevCommit commit_v2_0_1 = git.commit().setMessage("version 2.0.1").call();
git.tag().setName(TAG_2_0_1).setObjectId(commit_v2_0_1).call();
// Version 2.5
createAndCheckoutBranch(commit_v1_3, BRANCH_2_5);
git.merge().include(commit_v2_0_1).setCommit(false).setFastForward(FastForwardMode.NO_FF).call();
commit_v2_5 = git.commit().setMessage("version 2.5").call();
git.tag().setName(TAG_2_5).setObjectId(commit_v2_5).setAnnotated(false).call();
Ref ref_tag_2_5_annotated = git.tag().setName(TAG_2_5_ANNOTATED).setObjectId(commit_v2_5).setAnnotated(true).call();
RevTag tag_2_5_annotated = revWalk.parseTag(ref_tag_2_5_annotated.getObjectId());
git.tag().setName(TAG_2_5_ANNOTATED_TWICE).setObjectId(tag_2_5_annotated).setAnnotated(true).call();
In theory we could still unload the plugin even if the rename
failed. However, it would be reloaded on the next server startup,
which is probably not what the user expects.
SATD_ADDED
disablePlugins(Set)
public void disablePlugins(Set names)
if (!isRemoteAdminEnabled()) {
log.warn("Remote plugin administration is disabled," + " ignoring disablePlugins(" + names + ")");
return;
}
synchronized (this) {
for (String name : names) {
Plugin active = running.get(name);
if (active == null) {
continue;
}
log.info(String.format("Disabling plugin %s", active.getName()));
Path off = active.getSrcFile().resolveSibling(active.getSrcFile().getFileName() + ".disabled");
try {
Files.move(active.getSrcFile(), off);
} catch (IOException e) {
log.error("Failed to disable plugin", e);
// In theory we could still unload the plugin even if the rename
// failed. However, it would be reloaded on the next server startup,
// which is probably not what the user expects.
continue;
}
unloadPlugin(active);
try {
FileSnapshot snapshot = FileSnapshot.save(off.toFile());
Plugin offPlugin = loadPlugin(name, off, snapshot);
disabled.put(name, offPlugin);
} catch (Throwable e) {
// This shouldn't happen, as the plugin was loaded earlier.
log.warn(String.format("Cannot load disabled plugin %s", active.getName()), e.getCause());
}
}
cleanInBackground();
}
ChangeControl control = rsrc.getControl();
Change change = rsrc.getChange();
if (!control.canRebase()) {
throw new AuthException("rebase not permitted");
} else if (!change.getStatus().isOpen()) {
throw new ResourceConflictException("change is " + change.getStatus().name().toLowerCase());
} else if (!hasOneParent(rsrc.getPatchSet().getId())) {
throw new ResourceConflictException("cannot rebase merge commits or commit with no ancestor");
}
String baseRev = null;
if (input != null && input.base != null) {
String base = input.base.trim();
do {
if (base.equals("")) {
// remove existing dependency to other patch set
baseRev = change.getDest().get();
break;
}
ReviewDb db = dbProvider.get();
PatchSet basePatchSet = parseBase(base);
if (basePatchSet == null) {
throw new ResourceConflictException("base revision is missing: " + base);
} else if (!rsrc.getControl().isPatchVisible(basePatchSet, db)) {
throw new AuthException("base revision not accessible: " + base);
} else if (change.getId().equals(basePatchSet.getId().getParentKey())) {
throw new ResourceConflictException("cannot depend on self");
}
Change baseChange = db.changes().get(basePatchSet.getId().getParentKey());
if (baseChange != null) {
if (!baseChange.getProject().equals(change.getProject())) {
throw new ResourceConflictException("base change is in wrong project: " + baseChange.getProject());
} else if (!baseChange.getDest().equals(change.getDest())) {
throw new ResourceConflictException("base change is targetting wrong branch: " + baseChange.getDest());
} else if (baseChange.getStatus() == Status.ABANDONED) {
throw new ResourceConflictException("base change is abandoned: " + baseChange.getKey());
}
baseRev = basePatchSet.getRevision().get();
break;
}
} while (// just wanted to use the break statement
false);
}
try {
rebaseChange.get().rebase(rsrc.getChange(), rsrc.getPatchSet().getId(), rsrc.getUser(), baseRev);
} catch (InvalidChangeOperationException e) {
throw new ResourceConflictException(e.getMessage());
} catch (IOException e) {
throw new ResourceConflictException(e.getMessage());
} catch (NoSuchChangeException e) {
throw new ResourceNotFoundException(change.getId().toString());
}
return json.format(change.getId());
TODO(sbeller): why do we need to treat followup specially here?
SATD_ADDED
ActionInfo> toActionMap(ChangeControl)
private Map toActionMap(ChangeControl ctl)
Map out = new LinkedHashMap<>();
if (!ctl.getCurrentUser().isIdentifiedUser()) {
return out;
}
Provider userProvider = Providers.of(ctl.getCurrentUser());
for (UiAction.Description d : UiActions.from(changeViews, new ChangeResource(ctl), userProvider)) {
out.put(d.getId(), new ActionInfo(d));
}
// TODO(sbeller): why do we need to treat followup specially here?
if (ctl.getChange().getStatus().isOpen()) {
UiAction.Description descr = new UiAction.Description();
PrivateInternals_UiActionDescription.setId(descr, "followup");
PrivateInternals_UiActionDescription.setMethod(descr, "POST");
descr.setTitle("Create follow-up change");
out.put(descr.getId(), new ActionInfo(descr));
}
return out;
ignore non valid numbers
We don't want to popup another ugly dialog just to say
\"The number you've provided is invalid, try again\"
SATD_ADDED
gotoLine()
private Runnable gotoLine()
return new Runnable() {
@Override
public void run() {
String n = Window.prompt(EditConstants.I.gotoLineNumber(), "");
if (n != null) {
try {
int line = Integer.parseInt(n);
line--;
if (line >= 0) {
cm.scrollToLine(line);
}
} catch (NumberFormatException e) {
// ignore non valid numbers
// We don't want to popup another ugly dialog just to say
// "The number you've provided is invalid, try again"
}
}
}
};
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new MergeException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
chg = cd.change();
} catch (OrmException e) {
throw new MergeException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
toUpdate.add(chg);
continue;
}
PatchSet ps;
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new MergeException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
toUpdate.add(chg);
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
toUpdate.add(chg);
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
toUpdate.add(chg);
continue;
}
CodeReviewCommit commit;
try {
commit = (CodeReviewCommit) rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
toUpdate.add(chg);
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
toUpdate.add(chg);
continue;
}
if (branchTip != null) {
// If this commit is already merged its a bug in the queuing code
// that we got back here. Just mark it complete and move on. It's
// merged and that is all that mattered to the requestor.
//
try {
if (rw.isMergedInto(commit, branchTip)) {
logDebug("Revision {} of patch set {} is already merged", idstr, ps.getId());
commit.setStatusCode(CommitMergeStatus.ALREADY_MERGED);
try {
setMerged(chg, null);
} catch (OrmException e) {
logError("Cannot mark change " + chg.getId() + " merged", e);
}
continue;
}
} catch (IOException err) {
throw new MergeException("Cannot perform merge base test", err);
}
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
toUpdate.add(chg);
continue;
}
commit.add(canMergeFlag);
toMerge.put(submitType, commit);
toSubmit.put(submitType, chg);
}
return toSubmit;
TODO(dborowitz): Not sure whether this is injectable here.
SATD_ADDED
parseArguments(Parameters)
public final int parseArguments(final Parameters params) throws CmdLineException
final String token = params.getParameter(0);
final String[] tokens = token.split(",");
if (tokens.length != 3) {
throw new CmdLineException(owner, "change should be specified as " + ",,");
}
try {
final Change.Key key = Change.Key.parse(tokens[2]);
final Project.NameKey project = new Project.NameKey(tokens[0]);
final Branch.NameKey branch = new Branch.NameKey(project, "refs/heads/" + tokens[1]);
for (final ChangeData cd : queryProvider.get().byBranchKey(branch, key)) {
setter.addValue(cd.getId());
return 1;
}
} catch (IllegalArgumentException e) {
throw new CmdLineException(owner, "Change-Id is not valid");
} catch (OrmException e) {
throw new CmdLineException(owner, "Database error: " + e.getMessage());
}
throw new CmdLineException(owner, "\"" + token + "\": change not found");
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
TODO Technically the proper way to do this test is to use a
RevWalk on \"$id --not --all\" and test for an empty set. But
that is way slower than looking for a ref directly pointing
at the desired tip. We should always have a ref available.
TODO this is actually an error, the branch is gone but we
want to merge the issue. We can't safely do that if the
tip is not reachable.
ListMultimap toSubmit = ArrayListMultimap.create();
Map allRefs;
try {
allRefs = repo.getRefDatabase().getRefs(ALL);
} catch (IOException e) {
throw new MergeException(e.getMessage(), e);
}
Set tips = new HashSet<>();
for (Ref r : allRefs.values()) {
tips.add(r.getObjectId());
}
for (ChangeData cd : submitted) {
ChangeControl ctl;
Change chg;
try {
ctl = cd.changeControl();
chg = cd.change();
} catch (OrmException e) {
throw new MergeException("Failed to validate changes", e);
}
Change.Id changeId = cd.getId();
if (chg.currentPatchSetId() == null) {
logError("Missing current patch set on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
toUpdate.add(chg);
continue;
}
PatchSet ps;
try {
ps = cd.currentPatchSet();
} catch (OrmException e) {
throw new MergeException("Cannot query the database", e);
}
if (ps == null || ps.getRevision() == null || ps.getRevision().get() == null) {
logError("Missing patch set or revision on change " + changeId);
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
toUpdate.add(chg);
continue;
}
String idstr = ps.getRevision().get();
ObjectId id;
try {
id = ObjectId.fromString(idstr);
} catch (IllegalArgumentException iae) {
logError("Invalid revision on patch set " + ps.getId());
commits.put(changeId, CodeReviewCommit.noPatchSet(ctl));
toUpdate.add(chg);
continue;
}
if (!tips.contains(id)) {
// TODO Technically the proper way to do this test is to use a
// RevWalk on "$id --not --all" and test for an empty set. But
// that is way slower than looking for a ref directly pointing
// at the desired tip. We should always have a ref available.
//
// TODO this is actually an error, the branch is gone but we
// want to merge the issue. We can't safely do that if the
// tip is not reachable.
//
logError("Revision " + idstr + " of patch set " + ps.getId() + " is not contained in any ref");
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
toUpdate.add(chg);
continue;
}
CodeReviewCommit commit;
try {
commit = (CodeReviewCommit) rw.parseCommit(id);
} catch (IOException e) {
logError("Invalid commit " + idstr + " on patch set " + ps.getId(), e);
commits.put(changeId, CodeReviewCommit.revisionGone(ctl));
toUpdate.add(chg);
continue;
}
// TODO(dborowitz): Consider putting ChangeData in CodeReviewCommit.
commit.setControl(ctl);
commit.setPatchsetId(ps.getId());
commits.put(changeId, commit);
MergeValidators mergeValidators = mergeValidatorsFactory.create();
try {
mergeValidators.validatePreMerge(repo, commit, destProject, destBranch, ps.getId());
} catch (MergeValidationException mve) {
logDebug("Revision {} of patch set {} failed validation: {}", idstr, ps.getId(), mve.getStatus());
commit.setStatusCode(mve.getStatus());
toUpdate.add(chg);
continue;
}
if (branchTip != null) {
// If this commit is already merged its a bug in the queuing code
// that we got back here. Just mark it complete and move on. It's
// merged and that is all that mattered to the requestor.
//
try {
if (rw.isMergedInto(commit, branchTip)) {
logDebug("Revision {} of patch set {} is already merged", idstr, ps.getId());
commit.setStatusCode(CommitMergeStatus.ALREADY_MERGED);
try {
setMerged(chg, null);
} catch (OrmException e) {
logError("Cannot mark change " + chg.getId() + " merged", e);
}
continue;
}
} catch (IOException err) {
throw new MergeException("Cannot perform merge base test", err);
}
}
SubmitType submitType;
submitType = getSubmitType(commit.getControl(), ps);
if (submitType == null) {
logError("No submit type for revision " + idstr + " of patch set " + ps.getId());
commit.setStatusCode(CommitMergeStatus.NO_SUBMIT_TYPE);
toUpdate.add(chg);
continue;
}
commit.add(canMergeFlag);
toMerge.put(submitType, commit);
toSubmit.put(submitType, chg);
}
return toSubmit;
Always bump limit by 1, even if this results in exceeding the permitted
max for this user. The only way to see if there are more changes is to
ask for one more result from the query.
Always bump limit by 1, even if this results in exceeding the permitted
max for this user. The only way to see if there are more changes is to
ask for one more result from the query.
CLASS_OR_METHOD_CHANGED
queryChanges(List, List>)
private List queryChanges(List queryStrings, List> queries) throws OrmException, QueryParseException
Predicate visibleToMe = queryBuilder.is_visible();
int cnt = queries.size();
// Parse and rewrite all queries.
List limits = new ArrayList<>(cnt);
List> predicates = new ArrayList<>(cnt);
List sources = new ArrayList<>(cnt);
for (Predicate q : queries) {
q = Predicate.and(q, visibleToMe);
int limit = getEffectiveLimit(q);
limits.add(limit);
// Always bump limit by 1, even if this results in exceeding the permitted
// max for this user. The only way to see if there are more changes is to
// ask for one more result from the query.
Predicate s = queryRewriter.rewrite(q, start, limit + 1);
if (!(s instanceof ChangeDataSource)) {
q = Predicate.and(queryBuilder.status_open(), q);
s = queryRewriter.rewrite(q, start, limit);
}
if (!(s instanceof ChangeDataSource)) {
throw new QueryParseException("invalid query: " + s);
}
predicates.add(s);
// Don't trust QueryRewriter to have left the visible predicate.
// TODO(dborowitz): Probably we can.
AndSource a = new AndSource(ImmutableList.of(s, visibleToMe), start);
sources.add(a);
}
// Run each query asynchronously, if supported.
List> matches = new ArrayList<>(cnt);
for (ChangeDataSource s : sources) {
matches.add(s.read());
}
List out = new ArrayList<>(cnt);
for (int i = 0; i < cnt; i++) {
out.add(QueryResult.create(queryStrings != null ? queryStrings.get(i) : null, predicates.get(i), limits.get(i), matches.get(i).toList()));
}
return out;
Always bump limit by 1, even if this results in exceeding the permitted
max for this user. The only way to see if there are more changes is to
ask for one more result from the query.
Previously there was a bug where having a '1' in a refname would cause a
glob pattern's Levenshtein distance to decrease by 1. These two
patterns should be a Levenshtein distance of 12 from the both of the
refnames, where previously the 'branch1' refname would be a distance of
11 from 'refs/heads/abc/*'
SATD_ADDED
higherNumberOfTransitionsWins()
public void higherNumberOfTransitionsWins()
cmp = new MostSpecificComparator("refs/heads/x");
moreSpecificFirst("^refs/heads/[a-z].*", "refs/heads/*");
// Previously there was a bug where having a '1' in a refname would cause a
// glob pattern's Levenshtein distance to decrease by 1. These two
// patterns should be a Levenshtein distance of 12 from the both of the
// refnames, where previously the 'branch1' refname would be a distance of
// 11 from 'refs/heads/abc/*'
cmp = new MostSpecificComparator("refs/heads/abc/spam/branch2");
moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*");
cmp = new MostSpecificComparator("refs/heads/abc/spam/branch1");
moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*");
if (member.isActive()) {
IdentifiedUser user = identifiedUserFactory.create(member.getId());
// Does not account for draft status as a user might want to let a
// reviewer see a draft.
return control.forUser(user).isRefVisible();
}
return false;
TODO(davido): Clean this up by returning 404 when edit doesn't exist.
SATD_ADDED
apply(ChangeResource)
public BinaryResult apply(ChangeResource rsrc) throws AuthException, IOException, OrmException, NoSuchChangeException
Optional edit = editUtil.byChange(rsrc.getChange());
// TODO(davido): Clean this up by returning 404 when edit doesn't exist.
// Client should call GET /changes/{id}/revisions/current/commit in this
// case; or, to be consistent with GET content logic, the client could
// call directly the right endpoint.
String m = edit.isPresent() ? edit.get().getEditCommit().getFullMessage() : changeUtil.getMessage(rsrc.getChange());
return BinaryResult.create(m).base64();
Previously there was a bug where having a '1' in a refname would cause a
glob pattern's Levenshtein distance to decrease by 1. These two
patterns should be a Levenshtein distance of 12 from the both of the
refnames, where previously the 'branch1' refname would be a distance of
11 from 'refs/heads/abc/*'
SATD_ADDED
higherNumberOfTransitionsWins()
public void higherNumberOfTransitionsWins()
cmp = new MostSpecificComparator("refs/heads/x");
moreSpecificFirst("^refs/heads/[a-z].*", "refs/heads/*");
// Previously there was a bug where having a '1' in a refname would cause a
// glob pattern's Levenshtein distance to decrease by 1. These two
// patterns should be a Levenshtein distance of 12 from the both of the
// refnames, where previously the 'branch1' refname would be a distance of
// 11 from 'refs/heads/abc/*'
cmp = new MostSpecificComparator("refs/heads/abc/spam/branch2");
moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*");
cmp = new MostSpecificComparator("refs/heads/abc/spam/branch1");
moreSpecificFirst("^refs/heads/.*spam.*", "refs/heads/abc/*");
Only expose publish action when the edit is on top of current ps
SATD_ADDED
ActionInfo> fillActions(ChangeEdit)
private static Map fillActions(ChangeEdit edit)
Map actions = Maps.newTreeMap();
UiAction.Description descr = new UiAction.Description();
PrivateInternals_UiActionDescription.setId(descr, "/");
PrivateInternals_UiActionDescription.setMethod(descr, "DELETE");
descr.setTitle("Delete edit");
actions.put(descr.getId(), new ActionInfo(descr));
// Only expose publish action when the edit is on top of current ps
PatchSet.Id current = edit.getChange().currentPatchSetId();
PatchSet basePs = edit.getBasePatchSet();
if (basePs.getId().equals(current)) {
descr = new UiAction.Description();
PrivateInternals_UiActionDescription.setId(descr, "publish");
PrivateInternals_UiActionDescription.setMethod(descr, "POST");
descr.setTitle("Publish edit");
actions.put(descr.getId(), new ActionInfo(descr));
}
return actions;
For a file with content merge conflict that we produced a result
above on, collapse the file down to a single stage 0 with just
the blob content, and a randomly selected mode (the lowest stage,
which should be the merge base, or ours).
For a file with content merge conflict that we produced a result
above on, collapse the file down to a single stage 0 with just
the blob content, and a randomly selected mode (the lowest stage,
which should be the merge base, or ours).
String hash = b.name();
String refName = RefNames.REFS_CACHE_AUTOMERGE + hash.substring(0, 2) + "/" + hash.substring(2);
Ref ref = repo.getRef(refName);
if (ref != null && ref.getObjectId() != null) {
return rw.parseTree(ref.getObjectId());
}
ResolveMerger m = (ResolveMerger) mergeStrategy.newMerger(repo, true);
final ObjectInserter ins = repo.newObjectInserter();
try {
DirCache dc = DirCache.newInCore();
m.setDirCache(dc);
m.setObjectInserter(new ObjectInserter.Filter() {
@Override
protected ObjectInserter delegate() {
return ins;
}
@Override
public void flush() {
}
@Override
public void release() {
}
});
boolean couldMerge;
try {
couldMerge = m.merge(b.getParents());
} catch (IOException e) {
// It is not safe to continue further down in this method as throwing
// an exception most likely means that the merge tree was not created
// and m.getMergeResults() is empty. This would mean that all paths are
// unmerged and Gerrit UI would show all paths in the patch list.
log.warn("Error attempting automerge " + refName, e);
return null;
}
ObjectId treeId;
if (couldMerge) {
treeId = m.getResultTreeId();
} else {
RevCommit ours = b.getParent(0);
RevCommit theirs = b.getParent(1);
rw.parseBody(ours);
rw.parseBody(theirs);
String oursMsg = ours.getShortMessage();
String theirsMsg = theirs.getShortMessage();
String oursName = String.format("HEAD (%s %s)", ours.abbreviate(6).name(), oursMsg.substring(0, Math.min(oursMsg.length(), 60)));
String theirsName = String.format("BRANCH (%s %s)", theirs.abbreviate(6).name(), theirsMsg.substring(0, Math.min(theirsMsg.length(), 60)));
MergeFormatter fmt = new MergeFormatter();
Map> r = m.getMergeResults();
Map resolved = new HashMap<>();
for (Map.Entry> entry : r.entrySet()) {
MergeResult p = entry.getValue();
TemporaryBuffer buf = new TemporaryBuffer.LocalFile(10 * 1024 * 1024);
try {
fmt.formatMerge(buf, p, "BASE", oursName, theirsName, "UTF-8");
buf.close();
InputStream in = buf.openInputStream();
try {
resolved.put(entry.getKey(), ins.insert(Constants.OBJ_BLOB, buf.length(), in));
} finally {
in.close();
}
} finally {
buf.destroy();
}
}
DirCacheBuilder builder = dc.builder();
int cnt = dc.getEntryCount();
for (int i = 0; i < cnt; ) {
DirCacheEntry entry = dc.getEntry(i);
if (entry.getStage() == 0) {
builder.add(entry);
i++;
continue;
}
int next = dc.nextEntry(i);
String path = entry.getPathString();
DirCacheEntry res = new DirCacheEntry(path);
if (resolved.containsKey(path)) {
// For a file with content merge conflict that we produced a result
// above on, collapse the file down to a single stage 0 with just
// the blob content, and a randomly selected mode (the lowest stage,
// which should be the merge base, or ours).
res.setFileMode(entry.getFileMode());
res.setObjectId(resolved.get(path));
} else if (next == i + 1) {
// If there is exactly one stage present, shouldn't be a conflict...
res.setFileMode(entry.getFileMode());
res.setObjectId(entry.getObjectId());
} else if (next == i + 2) {
// Two stages suggests a delete/modify conflict. Pick the higher
// stage as the automatic result.
entry = dc.getEntry(i + 1);
res.setFileMode(entry.getFileMode());
res.setObjectId(entry.getObjectId());
} else {
// 3 stage conflict, no resolve above
// Punt on the 3-stage conflict and show the base, for now.
res.setFileMode(entry.getFileMode());
res.setObjectId(entry.getObjectId());
}
builder.add(res);
i = next;
}
builder.finish();
treeId = dc.writeTree(ins);
}
ins.flush();
if (save) {
RefUpdate update = repo.updateRef(refName);
update.setNewObjectId(treeId);
update.disableRefLog();
update.forceUpdate();
}
return rw.lookupTree(treeId);
} finally {
ins.release();
}
Edit is created on top of current patch set by deleting path.
Even if the latest patch set changed since the user triggered
the operation, deleting the whole file is probably still what
they intended.
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
public ListenableFuture rebuildAsync(final Change change, ListeningExecutorService executor, final BatchRefUpdate bru, final BatchRefUpdate bruForDrafts, final Repository changeRepo, final Repository allUsersRepo)
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
TODO(dborowitz): Smarter bucketing: pick a bucket start time T and
include all events up to T + TS_WINDOW_MS but no further.
Interleaving different authors complicates things.
SATD_ADDED
rebuildAsync(Change, ListeningExecutorService)
public ListenableFuture rebuildAsync(final Change change, ListeningExecutorService executor)
Repository repo = null;
// TODO - dborowitz: add NEW_CHANGE type for default.
ChangeKind kind = ChangeKind.REWORK;
// Trivial case: if we're on the first patch, we don't need to open
// the repository.
if (patch.getId().get() > 1) {
try {
ProjectState projectState = projectCache.checkedGet(change.getProject());
repo = repoManager.openRepository(change.getProject());
ChangeData cd = changeDataFactory.create(db, change);
Collection patchSetCollection = cd.patches();
PatchSet priorPs = patch;
for (PatchSet ps : patchSetCollection) {
if (ps.getId().get() < patch.getId().get() && (ps.getId().get() > priorPs.getId().get() || priorPs == patch)) {
// We only want the previous patch set, so walk until the last one
priorPs = ps;
}
}
// If we still think the previous patch is the current patch,
// we only have one patch set. Return the default.
// This can happen if a user creates a draft, uploads a second patch,
// and deletes the draft.
if (priorPs != patch) {
kind = cache.getChangeKind(projectState, repo, ObjectId.fromString(priorPs.getRevision().get()), ObjectId.fromString(patch.getRevision().get()));
}
} catch (IOException | OrmException e) {
// Do nothing; assume we have a complex change
log.warn("Unable to get change kind for patchSet " + patch.getPatchSetId() + "of change " + change.getChangeId(), e);
} finally {
if (repo != null) {
repo.close();
}
}
}
return kind;
If two or more projects contains \"query\" as substring create an
OrPredicate holding predicates for all these projects, otherwise if
only one contains that, return only that one predicate by itself.
private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del)
if (current.isEmpty() && ups.isEmpty()) {
// TODO Find another way to link reviewers to changes.
if (del.isEmpty()) {
// If no existing label is being set to 0, hack in the caller
// as a reviewer by picking the first server-wide LabelType.
PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs());
c.setGranted(timestamp);
c.cache(change);
ups.add(c);
} else {
// Pick a random label that is about to be deleted and keep it.
Iterator i = del.iterator();
PatchSetApproval c = i.next();
c.setValue((short) 0);
c.setGranted(timestamp);
c.cache(change);
i.remove();
ups.add(c);
}
}
private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del)
if (current.isEmpty() && ups.isEmpty()) {
// TODO Find another way to link reviewers to changes.
if (del.isEmpty()) {
// If no existing label is being set to 0, hack in the caller
// as a reviewer by picking the first server-wide LabelType.
PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs());
c.setGranted(timestamp);
c.cache(change);
ups.add(c);
} else {
// Pick a random label that is about to be deleted and keep it.
Iterator i = del.iterator();
PatchSetApproval c = i.next();
c.setValue((short) 0);
c.setGranted(timestamp);
c.cache(change);
i.remove();
ups.add(c);
}
}
private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del)
if (current.isEmpty() && ups.isEmpty()) {
// TODO Find another way to link reviewers to changes.
if (del.isEmpty()) {
// If no existing label is being set to 0, hack in the caller
// as a reviewer by picking the first server-wide LabelType.
PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs());
c.setGranted(timestamp);
ups.add(c);
} else {
// Pick a random label that is about to be deleted and keep it.
Iterator i = del.iterator();
PatchSetApproval c = i.next();
c.setValue((short) 0);
c.setGranted(timestamp);
i.remove();
ups.add(c);
}
}
private void forceCallerAsReviewer(RevisionResource rsrc, Map current, List ups, List del)
if (current.isEmpty() && ups.isEmpty()) {
// TODO Find another way to link reviewers to changes.
if (del.isEmpty()) {
// If no existing label is being set to 0, hack in the caller
// as a reviewer by picking the first server-wide LabelType.
PatchSetApproval c = new PatchSetApproval(new PatchSetApproval.Key(rsrc.getPatchSet().getId(), rsrc.getAccountId(), rsrc.getControl().getLabelTypes().getLabelTypes().get(0).getLabelId()), (short) 0, TimeUtil.nowTs());
c.setGranted(timestamp);
ups.add(c);
} else {
// Pick a random label that is about to be deleted and keep it.
Iterator i = del.iterator();
PatchSetApproval c = i.next();
c.setValue((short) 0);
c.setGranted(timestamp);
i.remove();
ups.add(c);
}
}
Don't use asserts from PushOneCommit so we can test the round-trip
through JSON instead of querying the DB directly.
SATD_ADDED
assertApproval(PushOneCommit.Result, int)
private void assertApproval(PushOneCommit.Result r, int expected) throws Exception
// Don't use asserts from PushOneCommit so we can test the round-trip
// through JSON instead of querying the DB directly.
LabelInfo cr = getChange(r).labels.get("Code-Review");
assertEquals(1, cr.all.size());
assertEquals("User", cr.all.get(0).name);
assertEquals(expected, cr.all.get(0).value.intValue());
// Flatten out existing approvals for this patch set based upon the current
// permissions. Once the change is closed the approvals are not updated at
// presentation view time, except for zero votes used to indicate a reviewer
// was added. So we need to make sure votes are accurate now. This way if
// permissions get modified in the future, historical records stay accurate.
PatchSetApproval submitter = null;
try {
List approvals = approvalsUtil.byPatchSet(db, notes, merged);
Set toDelete = Sets.newHashSetWithExpectedSize(approvals.size());
for (PatchSetApproval a : approvals) {
if (a.getValue() != 0) {
toDelete.add(a.getKey());
}
}
approvals = labelNormalizer.normalize(c, approvals);
for (PatchSetApproval a : approvals) {
toDelete.remove(a.getKey());
if (a.getValue() > 0 && a.isSubmit()) {
if (submitter == null || a.getGranted().compareTo(submitter.getGranted()) > 0) {
submitter = a;
}
}
}
// TODO(dborowitz): Store normalized labels in notedb.
db.patchSetApprovals().update(approvals);
db.patchSetApprovals().deleteKeys(toDelete);
} catch (NoSuchChangeException err) {
throw new OrmException(err);
}
return submitter;
Every comment appears in both side maps as a linked pair.
It is only necessary to search one side to find a comment
on either side of the editor pair.
SATD_ADDED
commentNav(CodeMirror, Direction)
return new Runnable() {
@Override
public void run() {
// Every comment appears in both side maps as a linked pair.
// It is only necessary to search one side to find a comment
// on either side of the editor pair.
SortedMap map = map(src.side());
int line = src.hasActiveLine() ? src.getLineNumber(src.getActiveLine()) + 1 : 0;
if (dir == Direction.NEXT) {
map = map.tailMap(line + 1);
if (map.isEmpty()) {
return;
}
line = map.firstKey();
} else {
map = map.headMap(line);
if (map.isEmpty()) {
return;
}
line = map.lastKey();
}
CommentGroup g = map.get(line);
if (g.getBoxCount() == 0) {
g = g.getPeer();
}
CodeMirror cm = g.getCm();
double y = cm.heightAtLine(g.getLine() - 1, "local");
cm.setCursor(LineCharacter.create(g.getLine() - 1));
cm.scrollToY(y - 0.5 * cm.getScrollbarV().getClientHeight());
cm.focus();
}
};
Runnable commentNav(final CodeMirror src, final Direction dir)
super.onInitUI();
addStyleName(Gerrit.RESOURCES.css().publishCommentsScreen());
approvalButtons = new ArrayList();
descBlock = new ChangeDescriptionBlock(null);
add(descBlock);
approvals = new ApprovalTable();
add(approvals);
final FormPanel form = new FormPanel();
final FlowPanel body = new FlowPanel();
form.setWidget(body);
form.addSubmitHandler(new FormPanel.SubmitHandler() {
@Override
public void onSubmit(final SubmitEvent event) {
event.cancel();
}
});
add(form);
approvalPanel = new FlowPanel();
body.add(approvalPanel);
initMessage(body);
draftsPanel = new FlowPanel();
body.add(draftsPanel);
final FlowPanel buttonRow = new FlowPanel();
buttonRow.setStyleName(Gerrit.RESOURCES.css().patchSetActions());
body.add(buttonRow);
send = new Button(Util.C.buttonPublishCommentsSend());
send.addClickHandler(this);
buttonRow.add(send);
submit = new Button(Util.C.buttonPublishSubmitSend());
submit.addClickHandler(this);
buttonRow.add(submit);
cancel = new Button(Util.C.buttonPublishCommentsCancel());
cancel.addClickHandler(this);
buttonRow.add(cancel);
GroupDescription.Basic group = groupsCollection.get().parseInternal(input.reviewer);
PostResult result = new PostResult();
if (!isLegalReviewerGroup(group.getGroupUUID())) {
result.error = MessageFormat.format(ChangeMessages.get().groupIsNotAllowed, group.getName());
return result;
}
Set reviewers = Sets.newLinkedHashSet();
ChangeControl control = rsrc.getControl();
Set members;
try {
members = groupMembersFactory.create(control.getCurrentUser()).listAccounts(group.getGroupUUID(), control.getProject().getNameKey());
} catch (NoSuchGroupException e) {
throw new UnprocessableEntityException(e.getMessage());
} catch (NoSuchProjectException e) {
throw new BadRequestException(e.getMessage());
}
// if maxAllowed is set to 0, it is allowed to add any number of
// reviewers
int maxAllowed = cfg.getInt("addreviewer", "maxAllowed", DEFAULT_MAX_REVIEWERS);
if (maxAllowed > 0 && members.size() > maxAllowed) {
result.error = MessageFormat.format(ChangeMessages.get().groupHasTooManyMembers, group.getName());
return result;
}
// if maxWithoutCheck is set to 0, we never ask for confirmation
int maxWithoutConfirmation = cfg.getInt("addreviewer", "maxWithoutConfirmation", DEFAULT_MAX_REVIEWERS_WITHOUT_CHECK);
if (!input.confirmed() && maxWithoutConfirmation > 0 && members.size() > maxWithoutConfirmation) {
result.confirm = true;
result.error = MessageFormat.format(ChangeMessages.get().groupManyMembersConfirmation, group.getName(), members.size());
return result;
}
for (Account member : members) {
if (member.isActive()) {
IdentifiedUser user = identifiedUserFactory.create(member.getId());
// Does not account for draft status as a user might want to let a
// reviewer see a draft.
if (control.forUser(user).isRefVisible()) {
reviewers.add(user);
}
}
}
addReviewers(rsrc, result, reviewers);
return result;
// TODO(sop): Switch to Dispatcher.toPatchSideBySide.
Change.Id c = ps.getParentKey();
return new StringBuilder().append("/c/").append(c.get()).append('/').append(ps.get()).append('/').append(KeyUtil.encode(info.path())).append(",cm").toString();
// TODO(sop): Switch to Dispatcher.toPatchSideBySide.
Change.Id c = ps.getParentKey();
return new StringBuilder().append("/c/").append(c.get()).append('/').append(ps.get()).append('/').append(KeyUtil.encode(info.path())).append(",cm").toString();
defer attribute here is to workaround IE running immediately.
defer attribute here is to workaround IE running immediately.
SATD_CHANGED
inject(Document)
public void inject(Document dom)
String moduleName = selector.getModuleName();
StringBuilder s = new StringBuilder();
s.append("\n");
s.append("function ").append(moduleName).append("(){");
s.append("var s,l,t");
s.append(",w=window");
s.append(",d=document");
s.append(",n='").append(moduleName).append("'");
s.append(",f=d.createElement('iframe')");
s.append(";");
// Callback to execute the module once both s and l are true.
//
s.append("function m(){");
s.append("if(s&&l){");
// Base path needs to be absolute. There isn't an easy way to do this
// other than forcing an image to load and then pulling the URL back.
//
s.append("var b,i=d.createElement('img');");
s.append("i.src=n+'/clear.cache.gif';");
s.append("b=i.src;");
s.append("b=b.substring(0,b.lastIndexOf('/')+1);");
// allow us to GC
s.append(moduleName).append("=null;");
s.append("f.contentWindow.gwtOnLoad(undefined,n,b);");
s.append("}");
s.append("}");
// Set s true when the module script has finished loading. The
// exact name here is known to the IFrameLinker and is called by
// the code in the iframe.
//
s.append(moduleName).append(".onScriptLoad=function(){");
s.append("s=1;m();");
s.append("};");
// Set l true when the browser has finished processing the iframe
// tag, and everything else on the page.
//
s.append(moduleName).append(".r=function(){");
s.append("l=1;m();");
s.append("};");
// Prevents mixed mode security in IE6/7.
s.append("f.src=\"javascript:''\";");
s.append("f.id=n;");
s.append("f.style.cssText" + "='position:absolute;width:0;height:0;border:none';");
s.append("f.tabIndex=-1;");
s.append("d.body.appendChild(f);");
// The src has to be set after the iframe is attached to the DOM to avoid
// refresh quirks in Safari. We have to use the location.replace trick to
// avoid FF2 refresh quirks.
//
s.append("f.contentWindow.location.replace(n+'/").append(cacheHTML).append("');");
// defer attribute here is to workaround IE running immediately.
//
s.append("d.write('